[llvm] [AMDGPU] SelectionDAG support for vector type set 0 to multiple sgpr64 (PR #128017)
Janek van Oirschot via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 20 07:27:41 PST 2025
https://github.com/JanekvO created https://github.com/llvm/llvm-project/pull/128017
Zeros seem to materialize into a sequence of `s_mov_b32`s, even when `s_mov_b64` is possible. First step to solving a worse case when 0 should be materialized into vector type vgprs as it first materializes into sgprs before `mov`ing into vgprs.
>From 34567eced88459d8f64a945af33bde84feb7f91e Mon Sep 17 00:00:00 2001
From: Janek van Oirschot <janek.vanoirschot at amd.com>
Date: Thu, 20 Feb 2025 14:46:47 +0000
Subject: [PATCH] [AMDGPU] SelectionDAG support for vector type materialization
of 0 to multiple sgpr64
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 55 ++
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 25 +-
.../CodeGen/AMDGPU/adjust-writemask-cse.ll | 17 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 274 +++++-----
.../AMDGPU/atomic-optimizer-strict-wqm.ll | 6 +-
.../AMDGPU/bitreverse-inline-immediates.ll | 2 +-
...der-no-live-segment-at-def-implicit-def.ll | 55 +-
llvm/test/CodeGen/AMDGPU/bug-cselect-b64.ll | 15 +-
llvm/test/CodeGen/AMDGPU/cluster_stores.ll | 17 +-
llvm/test/CodeGen/AMDGPU/collapse-endcf.ll | 41 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll | 14 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 34 +-
.../AMDGPU/fix-sgpr-copies-nondeterminism.ll | 8 +-
llvm/test/CodeGen/AMDGPU/flat-scratch.ll | 412 ++++++---------
.../AMDGPU/gfx-callable-return-types.ll | 14 +-
.../hazard-recognizer-src-shared-base.ll | 6 +-
.../identical-subrange-spill-infloop.ll | 495 ++++++------------
llvm/test/CodeGen/AMDGPU/imm.ll | 4 +-
.../CodeGen/AMDGPU/indirect-addressing-si.ll | 193 ++++---
...e92561-restore-undef-scc-verifier-error.ll | 18 +-
...474-need-live-out-undef-subregister-def.ll | 6 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 4 +-
.../CodeGen/AMDGPU/max-hard-clause-length.ll | 22 +-
.../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 4 +-
llvm/test/CodeGen/AMDGPU/mfma-loop.ll | 8 +-
.../AMDGPU/module-lds-false-sharing.ll | 14 +-
.../CodeGen/AMDGPU/no-dup-inst-prefetch.ll | 36 +-
.../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll | 34 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll | 106 +++-
...calc-one-successor-two-predecessors-bug.ll | 44 +-
...t_kill_i1_for_floation_point_comparison.ll | 10 +-
.../si-optimize-vgpr-live-range-dbg-instr.ll | 10 +-
.../CodeGen/AMDGPU/si-scheduler-exports.ll | 13 +-
llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll | 25 +-
.../AMDGPU/splitkit-getsubrangeformask.ll | 35 +-
.../AMDGPU/tuple-allocation-failure.ll | 72 ++-
.../test/CodeGen/AMDGPU/vni8-across-blocks.ll | 20 +-
llvm/test/CodeGen/AMDGPU/vopc_dpp.ll | 2 +-
.../CodeGen/AMDGPU/waterfall_kills_scc.ll | 21 +-
llvm/test/CodeGen/AMDGPU/wqm.ll | 36 +-
40 files changed, 1018 insertions(+), 1209 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 8e90754103ff1..c990df622175f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -634,6 +634,61 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
case ISD::BUILD_VECTOR: {
EVT VT = N->getValueType(0);
unsigned NumVectorElts = VT.getVectorNumElements();
+
+ auto IsSplatAllZeros = [this](SDNode *N) -> bool {
+ if (ISD::isConstantSplatVectorAllZeros(N))
+ return true;
+
+ // Types may have legalized by stripping the 16 bit multi-element vector
+ // into multiple BUILD_VECTORs. Peek through and see if it is all zeros
+ // regardless of what the legalizer did. Assumes cases along the lines of:
+ // v8i16 build_vector 0, 0, 0, 0, 0, 0, 0, 0
+ // -> legalizer ->
+ // t0 = v2i16 build_vector 0, 0
+ // t1 = bitcast t0 to i32
+ // v4i32 build_vector t1, t1, t1, t1
+ if (CurDAG->isSplatValue(SDValue(N, 0))) {
+ SDValue Op = peekThroughBitcasts(N->getOperand(0));
+ EVT InnerVT = Op.getValueType();
+ if (InnerVT.isVector() && Op.getOpcode() == ISD::BUILD_VECTOR &&
+ InnerVT.getVectorNumElements() == 2)
+ return ISD::isConstantSplatVectorAllZeros(Op.getNode());
+ }
+ return false;
+ };
+ if (IsSplatAllZeros(N)) {
+ unsigned FixedBitSize = VT.getFixedSizeInBits();
+ SDLoc DL(N);
+ if (FixedBitSize == 64) {
+ SDValue Set0 = {
+ CurDAG->getMachineNode(AMDGPU::S_MOV_B64_IMM_PSEUDO, DL, MVT::i64,
+ CurDAG->getTargetConstant(0, DL, MVT::i64)),
+ 0};
+ CurDAG->SelectNodeTo(N, AMDGPU::COPY, VT, Set0);
+ return;
+ } else if (NumVectorElts <= 32 && (FixedBitSize % 64 == 0)) {
+ SmallVector<SDValue, 32 * 2 + 1> Ops((FixedBitSize / 64) * 2 + 1);
+ SDValue Set0 = {
+ CurDAG->getMachineNode(AMDGPU::S_MOV_B64_IMM_PSEUDO, DL, MVT::i64,
+ CurDAG->getTargetConstant(0, DL, MVT::i64)),
+ 0};
+ unsigned RCID =
+ SIRegisterInfo::getSGPRClassForBitWidth(FixedBitSize)->getID();
+ Ops[0] = CurDAG->getTargetConstant(RCID, DL, MVT::i32);
+
+ for (unsigned i = 0, CurrentBitSize = FixedBitSize; CurrentBitSize != 0;
+ ++i, CurrentBitSize -= 64) {
+ unsigned SubRegs =
+ SIRegisterInfo::getSubRegFromChannel(i * 2, /*NumRegs=*/2);
+ Ops[i * 2 + 1] = Set0;
+ Ops[i * 2 + 2] = CurDAG->getTargetConstant(SubRegs, DL, MVT::i32);
+ }
+
+ CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, VT, Ops);
+ return;
+ }
+ }
+
if (VT.getScalarSizeInBits() == 16) {
if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index fa15e73bc31d5..3e7129f7bcd8f 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1035,6 +1035,9 @@ void SIFoldOperandsImpl::foldOperand(
getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
const DebugLoc &DL = UseMI->getDebugLoc();
MachineBasicBlock &MBB = *UseMI->getParent();
+ unsigned SubRegSize =
+ Defs.empty() ? 4
+ : TRI->getNumChannelsFromSubReg(Defs[0].second) * 4;
UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE));
for (unsigned I = UseMI->getNumOperands() - 1; I > 0; --I)
@@ -1043,17 +1046,29 @@ void SIFoldOperandsImpl::foldOperand(
MachineInstrBuilder B(*MBB.getParent(), UseMI);
DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
SmallSetVector<TargetInstrInfo::RegSubRegPair, 32> SeenAGPRs;
- for (unsigned I = 0; I < Size / 4; ++I) {
+ for (unsigned I = 0; I < Size / SubRegSize; ++I) {
MachineOperand *Def = Defs[I].first;
TargetInstrInfo::RegSubRegPair CopyToVGPR;
if (Def->isImm() &&
TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
int64_t Imm = Def->getImm();
- auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
- BuildMI(MBB, UseMI, DL,
- TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm);
- B.addReg(Tmp);
+ // Adding multiple immediates in case of 64 bit MOVs to agpr.
+ unsigned SubWordSize = SubRegSize / 4;
+ for (unsigned ImmIt = 0; ImmIt < SubWordSize; ++ImmIt) {
+ auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
+ unsigned SubRegs =
+ SIRegisterInfo::getSubRegFromChannel(I * SubWordSize + ImmIt);
+ BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64),
+ Tmp)
+ .addImm(Imm);
+ B.addReg(Tmp);
+ B.addImm(SubRegs);
+ }
+ // Avoid fallthrough that adds 'Defs[I].second' to B as we're
+ // technically pulling the SubReg apart from sgpr64 to
+ // agpr32:agpr32.
+ continue;
} else if (Def->isReg() && TRI->isAGPR(*MRI, Def->getReg())) {
auto Src = getRegSubRegPair(*Def);
Def->setIsKill(false);
diff --git a/llvm/test/CodeGen/AMDGPU/adjust-writemask-cse.ll b/llvm/test/CodeGen/AMDGPU/adjust-writemask-cse.ll
index 23a7bb6ece488..1d0a9f9585123 100644
--- a/llvm/test/CodeGen/AMDGPU/adjust-writemask-cse.ll
+++ b/llvm/test/CodeGen/AMDGPU/adjust-writemask-cse.ll
@@ -4,15 +4,14 @@
define float @test() {
; GFX10-LABEL: name: test
; GFX10: bb.0.bb:
- ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3, [[S_MOV_B32_]], %subreg.sub4, [[S_MOV_B32_]], %subreg.sub5, [[S_MOV_B32_]], %subreg.sub6, [[S_MOV_B32_]], %subreg.sub7
- ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX10-NEXT: [[IMAGE_LOAD_V2_V2_nsa_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_LOAD_V2_V2_nsa_gfx10 [[COPY]], [[COPY1]], killed [[REG_SEQUENCE]], 3, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 16, addrspace 8)
- ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[IMAGE_LOAD_V2_V2_nsa_gfx10_]].sub1
- ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[IMAGE_LOAD_V2_V2_nsa_gfx10_]].sub0
- ; GFX10-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY2]], 0, killed [[COPY3]], 0, 0, implicit $mode, implicit $exec
- ; GFX10-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_ADD_F32_e64_]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 0
+ ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B64_]], %subreg.sub2_sub3, [[S_MOV_B64_]], %subreg.sub4_sub5, [[S_MOV_B64_]], %subreg.sub6_sub7
+ ; GFX10-NEXT: [[V_MOV_B32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0
+ ; GFX10-NEXT: [[IMAGE_LOAD_V2_V2_nsa_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_LOAD_V2_V2_nsa_gfx10 [[V_MOV_B32_]], [[V_MOV_B32_]], killed [[REG_SEQUENCE]], 3, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 16, addrspace 8)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[IMAGE_LOAD_V2_V2_nsa_gfx10_]].sub1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[IMAGE_LOAD_V2_V2_nsa_gfx10_]].sub0
+ ; GFX10-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY]], 0, killed [[COPY1]], 0, 0, implicit $mode, implicit $exec
+ ; GFX10-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_ADD_F32_e64_]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: $vgpr0 = COPY [[V_ADD_F32_e64_1]]
; GFX10-NEXT: SI_RETURN implicit $vgpr0
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 4ce46bbaf45ac..02b17e6229b15 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -515,48 +515,47 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: global_load_ushort v16, v[0:1], off glc
; GFX908-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX908-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX908-NEXT: s_load_dword s7, s[8:9], 0x18
-; GFX908-NEXT: s_mov_b32 s6, 0
-; GFX908-NEXT: s_mov_b32 s9, s6
+; GFX908-NEXT: s_mov_b32 s7, 0
+; GFX908-NEXT: s_load_dword s8, s[8:9], 0x18
+; GFX908-NEXT: v_mov_b32_e32 v19, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX908-NEXT: s_sub_i32 s8, 0, s3
-; GFX908-NEXT: v_cvt_f32_f16_e32 v17, s7
-; GFX908-NEXT: v_mov_b32_e32 v19, 0
-; GFX908-NEXT: v_rcp_iflag_f32_e32 v2, v0
+; GFX908-NEXT: s_sub_i32 s6, 0, s3
+; GFX908-NEXT: v_cvt_f32_f16_e32 v17, s8
+; GFX908-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX908-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX908-NEXT: v_cvt_u32_f32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_mov_b32_e32 v1, 0
-; GFX908-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
-; GFX908-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX908-NEXT: v_readfirstlane_b32 s10, v2
-; GFX908-NEXT: s_mul_i32 s8, s8, s10
-; GFX908-NEXT: s_mul_hi_u32 s8, s10, s8
-; GFX908-NEXT: s_add_i32 s10, s10, s8
-; GFX908-NEXT: s_mul_hi_u32 s8, s2, s10
-; GFX908-NEXT: s_mul_i32 s10, s8, s3
-; GFX908-NEXT: s_sub_i32 s2, s2, s10
-; GFX908-NEXT: s_add_i32 s11, s8, 1
-; GFX908-NEXT: s_sub_i32 s10, s2, s3
+; GFX908-NEXT: v_readfirstlane_b32 s9, v2
+; GFX908-NEXT: s_mul_i32 s6, s6, s9
+; GFX908-NEXT: s_mul_hi_u32 s6, s9, s6
+; GFX908-NEXT: s_add_i32 s9, s9, s6
+; GFX908-NEXT: s_mul_hi_u32 s6, s2, s9
+; GFX908-NEXT: s_mul_i32 s9, s6, s3
+; GFX908-NEXT: s_sub_i32 s2, s2, s9
+; GFX908-NEXT: s_add_i32 s10, s6, 1
+; GFX908-NEXT: s_sub_i32 s9, s2, s3
; GFX908-NEXT: s_cmp_ge_u32 s2, s3
-; GFX908-NEXT: s_cselect_b32 s8, s11, s8
-; GFX908-NEXT: s_cselect_b32 s2, s10, s2
-; GFX908-NEXT: s_add_i32 s10, s8, 1
+; GFX908-NEXT: s_cselect_b32 s6, s10, s6
+; GFX908-NEXT: s_cselect_b32 s2, s9, s2
+; GFX908-NEXT: s_add_i32 s9, s6, 1
; GFX908-NEXT: s_cmp_ge_u32 s2, s3
-; GFX908-NEXT: s_cselect_b32 s8, s10, s8
-; GFX908-NEXT: s_lshr_b32 s7, s7, 16
-; GFX908-NEXT: v_cvt_f32_f16_e32 v18, s7
+; GFX908-NEXT: s_cselect_b32 s6, s9, s6
+; GFX908-NEXT: s_lshr_b32 s10, s8, 16
+; GFX908-NEXT: v_cvt_f32_f16_e32 v18, s10
+; GFX908-NEXT: s_lshl_b64 s[10:11], s[6:7], 5
; GFX908-NEXT: s_lshl_b64 s[2:3], s[0:1], 5
-; GFX908-NEXT: s_lshl_b64 s[12:13], s[8:9], 5
-; GFX908-NEXT: s_lshl_b64 s[10:11], s[4:5], 5
-; GFX908-NEXT: s_or_b32 s10, s10, 28
+; GFX908-NEXT: s_lshl_b64 s[8:9], s[4:5], 5
+; GFX908-NEXT: s_or_b32 s8, s8, 28
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s7, v16
; GFX908-NEXT: s_and_b32 s7, 0xffff, s7
; GFX908-NEXT: s_mul_i32 s1, s1, s7
-; GFX908-NEXT: s_mul_hi_u32 s9, s0, s7
+; GFX908-NEXT: s_mul_hi_u32 s12, s0, s7
; GFX908-NEXT: s_mul_i32 s0, s0, s7
-; GFX908-NEXT: s_add_i32 s1, s9, s1
-; GFX908-NEXT: s_lshl_b64 s[14:15], s[0:1], 5
+; GFX908-NEXT: s_add_i32 s1, s12, s1
+; GFX908-NEXT: s_lshl_b64 s[12:13], s[0:1], 5
; GFX908-NEXT: s_branch .LBB3_2
; GFX908-NEXT: .LBB3_1: ; %Flow20
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
@@ -565,59 +564,58 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: .LBB3_2: ; %bb9
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB3_5 Depth 2
-; GFX908-NEXT: s_mov_b64 s[16:17], -1
+; GFX908-NEXT: s_mov_b64 s[14:15], -1
; GFX908-NEXT: s_cbranch_scc0 .LBB3_10
; GFX908-NEXT: ; %bb.3: ; %bb14
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
; GFX908-NEXT: v_cmp_gt_i64_e64 s[0:1], s[4:5], -1
-; GFX908-NEXT: s_mov_b32 s7, s6
-; GFX908-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v4, s6
-; GFX908-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v6
-; GFX908-NEXT: v_mov_b32_e32 v6, s6
-; GFX908-NEXT: v_mov_b32_e32 v9, s7
-; GFX908-NEXT: v_mov_b32_e32 v5, s7
-; GFX908-NEXT: v_mov_b32_e32 v7, s7
-; GFX908-NEXT: v_mov_b32_e32 v8, s6
-; GFX908-NEXT: v_cmp_lt_i64_e64 s[16:17], s[4:5], 0
-; GFX908-NEXT: v_mov_b32_e32 v11, v5
-; GFX908-NEXT: s_mov_b64 s[18:19], s[10:11]
-; GFX908-NEXT: v_mov_b32_e32 v10, v4
+; GFX908-NEXT: v_cmp_lt_i64_e64 s[14:15], s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[0:1]
+; GFX908-NEXT: v_mov_b32_e32 v6, 0
+; GFX908-NEXT: v_mov_b32_e32 v8, 0
+; GFX908-NEXT: v_mov_b32_e32 v10, 0
+; GFX908-NEXT: v_mov_b32_e32 v5, 0
+; GFX908-NEXT: v_mov_b32_e32 v7, 0
+; GFX908-NEXT: v_mov_b32_e32 v9, 0
+; GFX908-NEXT: v_mov_b32_e32 v11, 0
+; GFX908-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v12
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s7, v2
-; GFX908-NEXT: v_readfirstlane_b32 s9, v3
+; GFX908-NEXT: v_readfirstlane_b32 s16, v3
; GFX908-NEXT: s_add_u32 s7, s7, 1
-; GFX908-NEXT: s_addc_u32 s9, s9, 0
-; GFX908-NEXT: s_mul_hi_u32 s20, s2, s7
-; GFX908-NEXT: s_mul_i32 s9, s2, s9
-; GFX908-NEXT: s_mul_i32 s21, s3, s7
-; GFX908-NEXT: s_add_i32 s9, s20, s9
+; GFX908-NEXT: s_addc_u32 s16, s16, 0
+; GFX908-NEXT: s_mul_hi_u32 s17, s2, s7
+; GFX908-NEXT: s_mul_i32 s16, s2, s16
+; GFX908-NEXT: s_mul_i32 s18, s3, s7
+; GFX908-NEXT: s_add_i32 s16, s17, s16
; GFX908-NEXT: s_mul_i32 s7, s2, s7
-; GFX908-NEXT: s_add_i32 s9, s9, s21
+; GFX908-NEXT: s_add_i32 s22, s16, s18
+; GFX908-NEXT: s_mov_b64 s[16:17], s[8:9]
; GFX908-NEXT: s_branch .LBB3_5
; GFX908-NEXT: .LBB3_4: ; %bb58
; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
; GFX908-NEXT: v_add_co_u32_sdwa v2, vcc, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX908-NEXT: s_add_u32 s18, s18, s14
-; GFX908-NEXT: v_cmp_lt_i64_e64 s[22:23], -1, v[2:3]
-; GFX908-NEXT: s_addc_u32 s19, s19, s15
-; GFX908-NEXT: s_mov_b64 s[20:21], 0
-; GFX908-NEXT: s_andn2_b64 vcc, exec, s[22:23]
+; GFX908-NEXT: s_add_u32 s16, s16, s12
+; GFX908-NEXT: v_cmp_lt_i64_e64 s[20:21], -1, v[2:3]
+; GFX908-NEXT: s_addc_u32 s17, s17, s13
+; GFX908-NEXT: s_mov_b64 s[18:19], 0
+; GFX908-NEXT: s_andn2_b64 vcc, exec, s[20:21]
; GFX908-NEXT: s_cbranch_vccz .LBB3_9
; GFX908-NEXT: .LBB3_5: ; %bb16
; GFX908-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT: s_add_u32 s20, s18, s7
-; GFX908-NEXT: s_addc_u32 s21, s19, s9
-; GFX908-NEXT: global_load_dword v21, v19, s[20:21] offset:-12 glc
+; GFX908-NEXT: s_add_u32 s18, s16, s7
+; GFX908-NEXT: s_addc_u32 s19, s17, s22
+; GFX908-NEXT: global_load_dword v21, v19, s[18:19] offset:-12 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_load_dword v20, v19, s[20:21] offset:-8 glc
+; GFX908-NEXT: global_load_dword v20, v19, s[18:19] offset:-8 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_load_dword v12, v19, s[20:21] offset:-4 glc
+; GFX908-NEXT: global_load_dword v12, v19, s[18:19] offset:-4 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_load_dword v12, v19, s[20:21] glc
+; GFX908-NEXT: global_load_dword v12, v19, s[18:19] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: ds_read_b64 v[12:13], v19
; GFX908-NEXT: ds_read_b64 v[14:15], v0
@@ -646,29 +644,29 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: v_add_f32_e32 v9, v9, v15
; GFX908-NEXT: v_add_f32_e32 v10, v10, v12
; GFX908-NEXT: v_add_f32_e32 v11, v11, v13
-; GFX908-NEXT: s_mov_b64 s[20:21], -1
+; GFX908-NEXT: s_mov_b64 s[18:19], -1
; GFX908-NEXT: s_branch .LBB3_4
; GFX908-NEXT: .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
-; GFX908-NEXT: s_mov_b64 s[20:21], s[16:17]
-; GFX908-NEXT: s_andn2_b64 vcc, exec, s[20:21]
+; GFX908-NEXT: s_mov_b64 s[18:19], s[14:15]
+; GFX908-NEXT: s_andn2_b64 vcc, exec, s[18:19]
; GFX908-NEXT: s_cbranch_vccz .LBB3_4
; GFX908-NEXT: ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX908-NEXT: ; implicit-def: $sgpr18_sgpr19
+; GFX908-NEXT: ; implicit-def: $sgpr16_sgpr17
; GFX908-NEXT: .LBB3_9: ; %loop.exit.guard
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_xor_b64 s[16:17], s[20:21], -1
+; GFX908-NEXT: s_xor_b64 s[14:15], s[18:19], -1
; GFX908-NEXT: .LBB3_10: ; %Flow19
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: s_mov_b64 s[0:1], -1
-; GFX908-NEXT: s_and_b64 vcc, exec, s[16:17]
+; GFX908-NEXT: s_and_b64 vcc, exec, s[14:15]
; GFX908-NEXT: s_cbranch_vccz .LBB3_1
; GFX908-NEXT: ; %bb.11: ; %bb12
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_add_u32 s4, s4, s8
+; GFX908-NEXT: s_add_u32 s4, s4, s6
; GFX908-NEXT: s_addc_u32 s5, s5, 0
-; GFX908-NEXT: s_add_u32 s10, s10, s12
-; GFX908-NEXT: s_addc_u32 s11, s11, s13
+; GFX908-NEXT: s_add_u32 s8, s8, s10
+; GFX908-NEXT: s_addc_u32 s9, s9, s11
; GFX908-NEXT: s_mov_b64 s[0:1], 0
; GFX908-NEXT: s_branch .LBB3_1
; GFX908-NEXT: .LBB3_12: ; %DummyReturnBlock
@@ -679,47 +677,46 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: global_load_ushort v18, v[0:1], off glc
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX90A-NEXT: s_load_dword s7, s[8:9], 0x18
-; GFX90A-NEXT: s_mov_b32 s6, 0
-; GFX90A-NEXT: s_mov_b32 s9, s6
+; GFX90A-NEXT: s_mov_b32 s7, 0
+; GFX90A-NEXT: s_load_dword s8, s[8:9], 0x18
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX90A-NEXT: s_sub_i32 s8, 0, s3
-; GFX90A-NEXT: v_mov_b32_e32 v19, 0
-; GFX90A-NEXT: v_rcp_iflag_f32_e32 v2, v0
+; GFX90A-NEXT: s_sub_i32 s6, 0, s3
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v2, s8
+; GFX90A-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX90A-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX90A-NEXT: v_cvt_u32_f32_e32 v3, v0
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], 0, 0
-; GFX90A-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
-; GFX90A-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v2, s7
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v3
-; GFX90A-NEXT: s_mul_i32 s8, s8, s10
-; GFX90A-NEXT: s_mul_hi_u32 s8, s10, s8
-; GFX90A-NEXT: s_add_i32 s10, s10, s8
-; GFX90A-NEXT: s_mul_hi_u32 s8, s2, s10
-; GFX90A-NEXT: s_mul_i32 s10, s8, s3
-; GFX90A-NEXT: s_sub_i32 s2, s2, s10
-; GFX90A-NEXT: s_add_i32 s11, s8, 1
-; GFX90A-NEXT: s_sub_i32 s10, s2, s3
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v3
+; GFX90A-NEXT: s_mul_i32 s6, s6, s9
+; GFX90A-NEXT: s_mul_hi_u32 s6, s9, s6
+; GFX90A-NEXT: s_add_i32 s9, s9, s6
+; GFX90A-NEXT: s_mul_hi_u32 s6, s2, s9
+; GFX90A-NEXT: s_mul_i32 s9, s6, s3
+; GFX90A-NEXT: s_sub_i32 s2, s2, s9
+; GFX90A-NEXT: s_add_i32 s10, s6, 1
+; GFX90A-NEXT: s_sub_i32 s9, s2, s3
; GFX90A-NEXT: s_cmp_ge_u32 s2, s3
-; GFX90A-NEXT: s_cselect_b32 s8, s11, s8
-; GFX90A-NEXT: s_cselect_b32 s2, s10, s2
-; GFX90A-NEXT: s_add_i32 s10, s8, 1
+; GFX90A-NEXT: s_cselect_b32 s6, s10, s6
+; GFX90A-NEXT: s_cselect_b32 s2, s9, s2
+; GFX90A-NEXT: s_add_i32 s9, s6, 1
; GFX90A-NEXT: s_cmp_ge_u32 s2, s3
-; GFX90A-NEXT: s_cselect_b32 s8, s10, s8
-; GFX90A-NEXT: s_lshr_b32 s7, s7, 16
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v3, s7
+; GFX90A-NEXT: s_cselect_b32 s6, s9, s6
+; GFX90A-NEXT: s_lshr_b32 s10, s8, 16
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v3, s10
+; GFX90A-NEXT: s_lshl_b64 s[10:11], s[6:7], 5
; GFX90A-NEXT: s_lshl_b64 s[2:3], s[0:1], 5
-; GFX90A-NEXT: s_lshl_b64 s[12:13], s[8:9], 5
-; GFX90A-NEXT: s_lshl_b64 s[10:11], s[4:5], 5
-; GFX90A-NEXT: s_or_b32 s10, s10, 28
+; GFX90A-NEXT: s_lshl_b64 s[8:9], s[4:5], 5
+; GFX90A-NEXT: s_or_b32 s8, s8, 28
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s7, v18
; GFX90A-NEXT: s_and_b32 s7, 0xffff, s7
; GFX90A-NEXT: s_mul_i32 s1, s1, s7
-; GFX90A-NEXT: s_mul_hi_u32 s9, s0, s7
+; GFX90A-NEXT: s_mul_hi_u32 s12, s0, s7
; GFX90A-NEXT: s_mul_i32 s0, s0, s7
-; GFX90A-NEXT: s_add_i32 s1, s9, s1
-; GFX90A-NEXT: s_lshl_b64 s[14:15], s[0:1], 5
+; GFX90A-NEXT: s_add_i32 s1, s12, s1
+; GFX90A-NEXT: s_lshl_b64 s[12:13], s[0:1], 5
; GFX90A-NEXT: s_branch .LBB3_2
; GFX90A-NEXT: .LBB3_1: ; %Flow20
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
@@ -728,60 +725,59 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: .LBB3_2: ; %bb9
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB3_5 Depth 2
-; GFX90A-NEXT: s_mov_b64 s[16:17], -1
+; GFX90A-NEXT: s_mov_b64 s[14:15], -1
; GFX90A-NEXT: s_cbranch_scc0 .LBB3_10
; GFX90A-NEXT: ; %bb.3: ; %bb14
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX90A-NEXT: v_cmp_gt_i64_e64 s[0:1], s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s7, s6
-; GFX90A-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[0:1]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v8
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_cmp_lt_i64_e64 s[16:17], s[4:5], 0
-; GFX90A-NEXT: s_mov_b64 s[18:19], s[10:11]
-; GFX90A-NEXT: v_pk_mov_b32 v[12:13], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[0:1]
+; GFX90A-NEXT: v_cmp_lt_i64_e64 s[14:15], s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], 0, 0
+; GFX90A-NEXT: s_mov_b64 s[16:17], s[8:9]
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], 0, 0
+; GFX90A-NEXT: v_pk_mov_b32 v[10:11], 0, 0
+; GFX90A-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v12
+; GFX90A-NEXT: v_pk_mov_b32 v[12:13], 0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s7, v4
-; GFX90A-NEXT: v_readfirstlane_b32 s9, v5
+; GFX90A-NEXT: v_readfirstlane_b32 s18, v5
; GFX90A-NEXT: s_add_u32 s7, s7, 1
-; GFX90A-NEXT: s_addc_u32 s9, s9, 0
-; GFX90A-NEXT: s_mul_hi_u32 s20, s2, s7
-; GFX90A-NEXT: s_mul_i32 s9, s2, s9
-; GFX90A-NEXT: s_mul_i32 s21, s3, s7
-; GFX90A-NEXT: s_add_i32 s9, s20, s9
+; GFX90A-NEXT: s_addc_u32 s18, s18, 0
+; GFX90A-NEXT: s_mul_hi_u32 s19, s2, s7
+; GFX90A-NEXT: s_mul_i32 s18, s2, s18
+; GFX90A-NEXT: s_mul_i32 s20, s3, s7
+; GFX90A-NEXT: s_add_i32 s18, s19, s18
; GFX90A-NEXT: s_mul_i32 s7, s2, s7
-; GFX90A-NEXT: s_add_i32 s9, s9, s21
+; GFX90A-NEXT: s_add_i32 s22, s18, s20
; GFX90A-NEXT: s_branch .LBB3_5
; GFX90A-NEXT: .LBB3_4: ; %bb58
; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
; GFX90A-NEXT: v_add_co_u32_sdwa v4, vcc, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX90A-NEXT: s_add_u32 s18, s18, s14
-; GFX90A-NEXT: s_addc_u32 s19, s19, s15
-; GFX90A-NEXT: v_cmp_lt_i64_e64 s[22:23], -1, v[4:5]
-; GFX90A-NEXT: s_mov_b64 s[20:21], 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[22:23]
+; GFX90A-NEXT: s_add_u32 s16, s16, s12
+; GFX90A-NEXT: s_addc_u32 s17, s17, s13
+; GFX90A-NEXT: v_cmp_lt_i64_e64 s[20:21], -1, v[4:5]
+; GFX90A-NEXT: s_mov_b64 s[18:19], 0
+; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[20:21]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_9
; GFX90A-NEXT: .LBB3_5: ; %bb16
; GFX90A-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT: s_add_u32 s20, s18, s7
-; GFX90A-NEXT: s_addc_u32 s21, s19, s9
-; GFX90A-NEXT: global_load_dword v21, v19, s[20:21] offset:-12 glc
+; GFX90A-NEXT: s_add_u32 s18, s16, s7
+; GFX90A-NEXT: s_addc_u32 s19, s17, s22
+; GFX90A-NEXT: global_load_dword v21, v19, s[18:19] offset:-12 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_load_dword v20, v19, s[20:21] offset:-8 glc
+; GFX90A-NEXT: global_load_dword v20, v19, s[18:19] offset:-8 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_load_dword v14, v19, s[20:21] offset:-4 glc
+; GFX90A-NEXT: global_load_dword v14, v19, s[18:19] offset:-4 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_load_dword v14, v19, s[20:21] glc
+; GFX90A-NEXT: global_load_dword v14, v19, s[18:19] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ds_read_b64 v[14:15], v19
; GFX90A-NEXT: ds_read_b64 v[16:17], v0
; GFX90A-NEXT: s_and_b64 vcc, exec, s[0:1]
-; GFX90A-NEXT: ; kill: killed $sgpr20 killed $sgpr21
+; GFX90A-NEXT: ; kill: killed $sgpr18 killed $sgpr19
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_cbranch_vccnz .LBB3_7
; GFX90A-NEXT: ; %bb.6: ; %bb51
@@ -798,29 +794,29 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[26:27]
; GFX90A-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[16:17]
; GFX90A-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[14:15]
-; GFX90A-NEXT: s_mov_b64 s[20:21], -1
+; GFX90A-NEXT: s_mov_b64 s[18:19], -1
; GFX90A-NEXT: s_branch .LBB3_4
; GFX90A-NEXT: .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
-; GFX90A-NEXT: s_mov_b64 s[20:21], s[16:17]
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[20:21]
+; GFX90A-NEXT: s_mov_b64 s[18:19], s[14:15]
+; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[18:19]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_4
; GFX90A-NEXT: ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX90A-NEXT: ; implicit-def: $sgpr18_sgpr19
+; GFX90A-NEXT: ; implicit-def: $sgpr16_sgpr17
; GFX90A-NEXT: .LBB3_9: ; %loop.exit.guard
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_xor_b64 s[16:17], s[20:21], -1
+; GFX90A-NEXT: s_xor_b64 s[14:15], s[18:19], -1
; GFX90A-NEXT: .LBB3_10: ; %Flow19
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: s_mov_b64 s[0:1], -1
-; GFX90A-NEXT: s_and_b64 vcc, exec, s[16:17]
+; GFX90A-NEXT: s_and_b64 vcc, exec, s[14:15]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_1
; GFX90A-NEXT: ; %bb.11: ; %bb12
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_add_u32 s4, s4, s8
+; GFX90A-NEXT: s_add_u32 s4, s4, s6
; GFX90A-NEXT: s_addc_u32 s5, s5, 0
-; GFX90A-NEXT: s_add_u32 s10, s10, s12
-; GFX90A-NEXT: s_addc_u32 s11, s11, s13
+; GFX90A-NEXT: s_add_u32 s8, s8, s10
+; GFX90A-NEXT: s_addc_u32 s9, s9, s11
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_branch .LBB3_1
; GFX90A-NEXT: .LBB3_12: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
index e03c9ca34b825..13f5b6598f6fa 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
@@ -11,7 +11,7 @@ define amdgpu_ps void @main(i32 %arg) {
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s1, exec_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b64 s[4:5], 0
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: s_branch .LBB0_2
; GFX10-NEXT: .LBB0_1: ; in Loop: Header=BB0_2 Depth=1
@@ -31,9 +31,7 @@ define amdgpu_ps void @main(i32 %arg) {
; GFX10-NEXT: s_and_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB0_1
; GFX10-NEXT: ; %bb.4: ; in Loop: Header=BB0_2 Depth=1
-; GFX10-NEXT: s_mov_b32 s5, s4
-; GFX10-NEXT: s_mov_b32 s6, s4
-; GFX10-NEXT: s_mov_b32 s7, s4
+; GFX10-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX10-NEXT: buffer_atomic_and v0, off, s[4:7], 0
; GFX10-NEXT: s_branch .LBB0_1
; GFX10-NEXT: .LBB0_5: ; %bb8
diff --git a/llvm/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll b/llvm/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll
index de2e25651271a..e0f9855ef2741 100644
--- a/llvm/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll
@@ -16,7 +16,7 @@ define amdgpu_kernel void @materialize_0_i32(ptr addrspace(1) %out) {
; GCN-LABEL: {{^}}materialize_0_i64:
; GCN: v_mov_b32_e32 v[[LOK:[0-9]+]], 0{{$}}
-; GCN: v_mov_b32_e32 v[[HIK:[0-9]+]], v[[LOK]]{{$}}
+; GCN: v_mov_b32_e32 v[[HIK:[0-9]+]], 0{{$}}
; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v[[[LOK]]:[[HIK]]]
define amdgpu_kernel void @materialize_0_i64(ptr addrspace(1) %out) {
store i64 0, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
index f9ffa5ae57f3e..4627097f4b1e7 100644
--- a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
@@ -9,44 +9,36 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: s_addc_u32 s13, s13, 0
; CHECK-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; CHECK-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; CHECK-NEXT: s_load_dwordx8 s[36:43], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx8 s[20:27], s[8:9], 0x0
; CHECK-NEXT: s_add_u32 s0, s0, s17
; CHECK-NEXT: s_addc_u32 s1, s1, 0
-; CHECK-NEXT: s_mov_b32 s12, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_cmp_lg_u32 s40, 0
+; CHECK-NEXT: s_cmp_lg_u32 s24, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_8
; CHECK-NEXT: ; %bb.1: ; %if.end13.i.i
-; CHECK-NEXT: s_cmp_eq_u32 s42, 0
-; CHECK-NEXT: s_cbranch_scc1 .LBB0_4
+; CHECK-NEXT: s_cmp_eq_u32 s26, 0
+; CHECK-NEXT: s_cbranch_scc1 .LBB0_6
; CHECK-NEXT: ; %bb.2: ; %if.else251.i.i
-; CHECK-NEXT: s_cmp_lg_u32 s43, 0
+; CHECK-NEXT: s_cmp_lg_u32 s27, 0
; CHECK-NEXT: s_mov_b32 s17, 0
; CHECK-NEXT: s_cselect_b32 s12, -1, 0
; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s12
-; CHECK-NEXT: s_cbranch_vccz .LBB0_5
+; CHECK-NEXT: s_cbranch_vccz .LBB0_4
; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: s_mov_b32 s36, 0
+; CHECK-NEXT: s_mov_b32 s20, 0
; CHECK-NEXT: s_andn2_b32 vcc_lo, exec_lo, s12
-; CHECK-NEXT: s_cbranch_vccz .LBB0_6
-; CHECK-NEXT: s_branch .LBB0_7
-; CHECK-NEXT: .LBB0_4:
-; CHECK-NEXT: s_mov_b32 s14, s12
-; CHECK-NEXT: s_mov_b32 s15, s12
-; CHECK-NEXT: s_mov_b32 s13, s12
-; CHECK-NEXT: s_mov_b64 s[38:39], s[14:15]
-; CHECK-NEXT: s_mov_b64 s[36:37], s[12:13]
+; CHECK-NEXT: s_cbranch_vccz .LBB0_5
; CHECK-NEXT: s_branch .LBB0_7
-; CHECK-NEXT: .LBB0_5: ; %if.then263.i.i
-; CHECK-NEXT: v_cmp_lt_f32_e64 s12, s41, 0
-; CHECK-NEXT: s_mov_b32 s36, 1.0
+; CHECK-NEXT: .LBB0_4: ; %if.then263.i.i
+; CHECK-NEXT: v_cmp_lt_f32_e64 s12, s25, 0
+; CHECK-NEXT: s_mov_b32 s20, 1.0
; CHECK-NEXT: s_mov_b32 s17, 0x7fc00000
-; CHECK-NEXT: s_mov_b32 s37, s36
-; CHECK-NEXT: s_mov_b32 s38, s36
-; CHECK-NEXT: s_mov_b32 s39, s36
+; CHECK-NEXT: s_mov_b32 s21, s20
+; CHECK-NEXT: s_mov_b32 s22, s20
+; CHECK-NEXT: s_mov_b32 s23, s20
; CHECK-NEXT: s_andn2_b32 vcc_lo, exec_lo, s12
; CHECK-NEXT: s_cbranch_vccnz .LBB0_7
-; CHECK-NEXT: .LBB0_6: ; %if.end273.i.i
+; CHECK-NEXT: .LBB0_5: ; %if.end273.i.i
; CHECK-NEXT: s_add_u32 s12, s8, 40
; CHECK-NEXT: s_addc_u32 s13, s9, 0
; CHECK-NEXT: s_getpc_b64 s[18:19]
@@ -55,7 +47,7 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 20, v2
; CHECK-NEXT: s_load_dwordx2 s[18:19], s[18:19], 0x0
; CHECK-NEXT: v_lshlrev_b32_e32 v3, 10, v1
-; CHECK-NEXT: v_add_f32_e64 v1, s17, s36
+; CHECK-NEXT: v_add_f32_e64 v1, s17, s20
; CHECK-NEXT: s_mov_b64 s[34:35], s[8:9]
; CHECK-NEXT: s_mov_b64 s[8:9], s[12:13]
; CHECK-NEXT: s_mov_b32 s12, s14
@@ -65,13 +57,12 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: s_mov_b32 s13, s15
; CHECK-NEXT: s_mov_b32 s14, s16
-; CHECK-NEXT: s_mov_b32 s36, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[18:19]
; CHECK-NEXT: s_mov_b64 s[8:9], s[34:35]
-; CHECK-NEXT: s_mov_b32 s37, s36
-; CHECK-NEXT: s_mov_b32 s38, s36
-; CHECK-NEXT: s_mov_b32 s39, s36
+; CHECK-NEXT: .LBB0_6: ; %if.end294.i.i
+; CHECK-NEXT: s_mov_b64 s[20:21], 0
+; CHECK-NEXT: s_mov_b64 s[22:23], s[20:21]
; CHECK-NEXT: .LBB0_7: ; %if.end294.i.i
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], 0 offset:12
@@ -80,11 +71,11 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], 0
; CHECK-NEXT: .LBB0_8: ; %kernel_direct_lighting.exit
; CHECK-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x20
-; CHECK-NEXT: v_mov_b32_e32 v0, s36
+; CHECK-NEXT: v_mov_b32_e32 v0, s20
; CHECK-NEXT: v_mov_b32_e32 v4, 0
-; CHECK-NEXT: v_mov_b32_e32 v1, s37
-; CHECK-NEXT: v_mov_b32_e32 v2, s38
-; CHECK-NEXT: v_mov_b32_e32 v3, s39
+; CHECK-NEXT: v_mov_b32_e32 v1, s21
+; CHECK-NEXT: v_mov_b32_e32 v2, s22
+; CHECK-NEXT: v_mov_b32_e32 v3, s23
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/bug-cselect-b64.ll b/llvm/test/CodeGen/AMDGPU/bug-cselect-b64.ll
index f6fc69a6e3e47..ba58d82ef8e60 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-cselect-b64.ll
+++ b/llvm/test/CodeGen/AMDGPU/bug-cselect-b64.ll
@@ -4,17 +4,14 @@
define amdgpu_cs <2 x i32> @f() {
; CHECK-LABEL: f:
; CHECK: ; %bb.0: ; %bb
-; CHECK-NEXT: s_mov_b32 s4, 0
-; CHECK-NEXT: s_mov_b32 s5, s4
-; CHECK-NEXT: s_mov_b32 s6, s4
-; CHECK-NEXT: s_mov_b32 s7, s4
-; CHECK-NEXT: s_mov_b32 s0, s4
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: s_mov_b64 s[6:7], s[4:5]
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; CHECK-NEXT: s_mov_b32 s1, s4
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, s[0:1], v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v1, s4
-; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; CHECK-NEXT: v_mov_b32_e32 v1, s0
; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; CHECK-NEXT: v_readfirstlane_b32 s0, v0
; CHECK-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
index 986dd8a046424..ddb81ea7af764 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
@@ -446,19 +446,20 @@ define amdgpu_ps void @cluster_image_sample(<8 x i32> inreg %src, <4 x i32> inre
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: v_cvt_f32_i32_e32 v4, v0
; GFX11-NEXT: v_cvt_f32_i32_e32 v5, v1
+; GFX11-NEXT: v_mov_b32_e32 v6, 1.0
; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_mov_b32 v6, 1.0 :: v_dual_add_f32 v11, 2.0, v5
-; GFX11-NEXT: v_dual_add_f32 v9, 1.0, v5 :: v_dual_add_f32 v8, 1.0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_add_f32 v10, 2.0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_add_f32 v11, 2.0, v4
+; GFX11-NEXT: v_dual_add_f32 v8, 1.0, v4 :: v_dual_add_f32 v9, 1.0, v5
+; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_add_f32 v12, 2.0, v5
; GFX11-NEXT: v_mov_b32_e32 v7, v6
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: image_sample_d v[2:5], [v8, v9, v2, v2, v[2:3]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT: image_sample_d v[6:9], [v10, v11, v6, v6, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT: image_sample_d v[2:5], [v8, v9, v10, v10, v[2:3]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT: image_sample_d v[6:9], [v11, v12, v6, v6, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v7
; GFX11-NEXT: v_dual_add_f32 v5, v5, v9 :: v_dual_add_f32 v4, v4, v8
-; GFX11-NEXT: v_dual_add_f32 v3, v3, v7 :: v_dual_add_f32 v2, v2, v6
+; GFX11-NEXT: v_add_f32_e32 v2, v2, v6
; GFX11-NEXT: image_store v[2:5], v[0:1], s[12:19] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
; GFX11-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
index 50c9c0cb64ccd..2763e88e2f00a 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -981,14 +981,14 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: s_movk_i32 s4, 0x207
; GCN-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
-; GCN-NEXT: s_mov_b32 s8, 0
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
; GCN-NEXT: s_mov_b64 s[12:13], 0
+; GCN-NEXT: s_mov_b64 s[8:9], 0
; GCN-NEXT: s_mov_b64 s[6:7], 0
; GCN-NEXT: s_branch .LBB5_3
; GCN-NEXT: .LBB5_1: ; %Flow
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
-; GCN-NEXT: s_or_b64 exec, exec, s[10:11]
+; GCN-NEXT: s_or_b64 exec, exec, s[16:17]
; GCN-NEXT: .LBB5_2: ; %bb10
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
; GCN-NEXT: s_or_b64 exec, exec, s[14:15]
@@ -1006,9 +1006,7 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-NEXT: ; %bb.4: ; %bb2
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: s_mov_b32 s9, s8
-; GCN-NEXT: s_mov_b32 s10, s8
-; GCN-NEXT: s_mov_b32 s11, s8
+; GCN-NEXT: s_mov_b64 s[10:11], s[8:9]
; GCN-NEXT: v_mov_b32_e32 v0, s8
; GCN-NEXT: v_mov_b32_e32 v1, s9
; GCN-NEXT: v_mov_b32_e32 v2, s10
@@ -1024,11 +1022,11 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-NEXT: v_mov_b32_e32 v1, s9
; GCN-NEXT: v_mov_b32_e32 v2, s10
; GCN-NEXT: v_mov_b32_e32 v3, s11
-; GCN-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
+; GCN-NEXT: s_and_saveexec_b64 s[16:17], s[6:7]
; GCN-NEXT: s_cbranch_execz .LBB5_1
; GCN-NEXT: ; %bb.6: ; %bb8
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
-; GCN-NEXT: s_mov_b32 s9, s8
+; GCN-NEXT: s_mov_b64 s[10:11], s[8:9]
; GCN-NEXT: v_mov_b32_e32 v0, s8
; GCN-NEXT: v_mov_b32_e32 v1, s9
; GCN-NEXT: v_mov_b32_e32 v2, s10
@@ -1111,11 +1109,9 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v0, s6
; GCN-O0-NEXT: v_writelane_b32 v6, s4, 8
; GCN-O0-NEXT: v_writelane_b32 v6, s5, 9
-; GCN-O0-NEXT: s_mov_b32 s4, 0
-; GCN-O0-NEXT: s_mov_b32 s8, s4
-; GCN-O0-NEXT: s_mov_b32 s9, s4
-; GCN-O0-NEXT: s_mov_b32 s10, s4
-; GCN-O0-NEXT: s_mov_b32 s11, s4
+; GCN-O0-NEXT: s_mov_b64 s[4:5], 0
+; GCN-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GCN-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
; GCN-O0-NEXT: v_mov_b32_e32 v0, s8
; GCN-O0-NEXT: v_mov_b32_e32 v1, s9
; GCN-O0-NEXT: v_mov_b32_e32 v2, s10
@@ -1145,10 +1141,9 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_mov_b32 s4, 0
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_cmp_lt_f32_e64 s[6:7], v0, s4
-; GCN-O0-NEXT: s_mov_b32 s8, s4
-; GCN-O0-NEXT: s_mov_b32 s9, s4
-; GCN-O0-NEXT: s_mov_b32 s10, s4
-; GCN-O0-NEXT: s_mov_b32 s11, s4
+; GCN-O0-NEXT: s_mov_b64 s[4:5], 0
+; GCN-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GCN-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
; GCN-O0-NEXT: v_mov_b32_e32 v0, s8
; GCN-O0-NEXT: v_mov_b32_e32 v1, s9
; GCN-O0-NEXT: v_mov_b32_e32 v2, s10
@@ -1168,17 +1163,9 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_cbranch_execz .LBB5_6
; GCN-O0-NEXT: ; %bb.4: ; %bb8
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_mov_b32 s10, 0
-; GCN-O0-NEXT: ; implicit-def: $sgpr4
-; GCN-O0-NEXT: ; implicit-def: $sgpr5
-; GCN-O0-NEXT: ; implicit-def: $sgpr9
-; GCN-O0-NEXT: ; implicit-def: $sgpr5
-; GCN-O0-NEXT: ; implicit-def: $sgpr8
-; GCN-O0-NEXT: ; implicit-def: $sgpr5
-; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
-; GCN-O0-NEXT: s_mov_b32 s5, s10
-; GCN-O0-NEXT: s_mov_b32 s6, s9
-; GCN-O0-NEXT: s_mov_b32 s7, s8
+; GCN-O0-NEXT: s_mov_b64 s[8:9], 0
+; GCN-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
+; GCN-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
; GCN-O0-NEXT: s_waitcnt expcnt(1)
; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
; GCN-O0-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index b4dbe0e7be924..84d312ea2e2a1 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -2679,7 +2679,7 @@ define amdgpu_kernel void @s_test_canonicalize_undef_v4f16(ptr addrspace(1) %out
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
@@ -2690,9 +2690,10 @@ define amdgpu_kernel void @s_test_canonicalize_undef_v4f16(ptr addrspace(1) %out
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; CI-LABEL: s_test_canonicalize_undef_v4f16:
@@ -2700,8 +2701,8 @@ define amdgpu_kernel void @s_test_canonicalize_undef_v4f16(ptr addrspace(1) %out
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 0
; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: v_mov_b32_e32 v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
@@ -2710,10 +2711,9 @@ define amdgpu_kernel void @s_test_canonicalize_undef_v4f16(ptr addrspace(1) %out
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b64 v0, v[0:1], s[0:1]
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
%canonicalized = call <4 x half> @llvm.canonicalize.v4f16(<4 x half> undef)
store <4 x half> %canonicalized, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index 4e12a30c6f6f4..996c485fe9a8d 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -1363,7 +1363,7 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
; GFX678: ; %bb.0:
; GFX678-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX678-NEXT: v_mov_b32_e32 v0, 0
-; GFX678-NEXT: v_mov_b32_e32 v1, v0
+; GFX678-NEXT: v_mov_b32_e32 v1, 0
; GFX678-NEXT: s_waitcnt lgkmcnt(0)
; GFX678-NEXT: v_mov_b32_e32 v3, s1
; GFX678-NEXT: v_mov_b32_e32 v2, s0
@@ -1374,29 +1374,28 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_fold_canonicalize_p0_f64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b64 v0, v[0:1], s[0:1]
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_fold_canonicalize_p0_f64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b64 v0, v[0:1], s[0:1]
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%canonicalized = call double @llvm.canonicalize.f64(double 0.0)
store double %canonicalized, ptr addrspace(1) %out
@@ -1574,7 +1573,7 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
; GFX678: ; %bb.0:
; GFX678-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX678-NEXT: v_mov_b32_e32 v0, 0
-; GFX678-NEXT: v_mov_b32_e32 v1, v0
+; GFX678-NEXT: v_mov_b32_e32 v1, 0
; GFX678-NEXT: s_waitcnt lgkmcnt(0)
; GFX678-NEXT: v_mov_b32_e32 v3, s1
; GFX678-NEXT: v_mov_b32_e32 v2, s0
@@ -1585,29 +1584,28 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b64 v0, v[0:1], s[0:1]
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v1, v0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b64 v0, v[0:1], s[0:1]
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%canonicalized = call double @llvm.canonicalize.f64(double bitcast (i64 4503599627370495 to double))
store double %canonicalized, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
index 67cfe52482542..20c88f99ba7d2 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
@@ -5,7 +5,6 @@ define amdgpu_gs void @f(i32 inreg %arg, i32 %arg1, i32 %arg2) {
; CHECK-LABEL: f:
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: s_cmp_eq_u32 s0, 0
-; CHECK-NEXT: s_mov_b32 s0, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_2
; CHECK-NEXT: ; %bb.1: ; %bb3
; CHECK-NEXT: v_mov_b32_e32 v5, v0
@@ -15,10 +14,9 @@ define amdgpu_gs void @f(i32 inreg %arg, i32 %arg1, i32 %arg2) {
; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: .LBB0_3: ; %bb4
; CHECK-NEXT: v_mov_b32_e32 v6, 0
-; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_mov_b32 s2, s0
-; CHECK-NEXT: s_mov_b32 s3, s0
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT: s_mov_b64 s[2:3], s[0:1]
; CHECK-NEXT: v_mov_b32_e32 v7, v6
; CHECK-NEXT: v_mov_b32_e32 v8, v6
; CHECK-NEXT: v_mov_b32_e32 v2, v6
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index c39a03ee8008c..06f51ea611761 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -13,16 +13,15 @@
define amdgpu_kernel void @zero_init_kernel() {
; GFX9-LABEL: zero_init_kernel:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_add_u32 flat_scratch_lo, s8, s13
-; GFX9-NEXT: s_mov_b32 s1, s0
-; GFX9-NEXT: s_mov_b32 s2, s0
-; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: s_mov_b32 s0, 0
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:48
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:32
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:16
@@ -35,10 +34,8 @@ define amdgpu_kernel void @zero_init_kernel() {
; GFX10-NEXT: s_addc_u32 s9, s9, 0
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -51,11 +48,9 @@ define amdgpu_kernel void @zero_init_kernel() {
;
; GFX11-LABEL: zero_init_kernel:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_clause 0x3
@@ -67,11 +62,9 @@ define amdgpu_kernel void @zero_init_kernel() {
;
; GFX12-LABEL: zero_init_kernel:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_clause 0x3
@@ -86,18 +79,17 @@ define amdgpu_kernel void @zero_init_kernel() {
; GFX9-PAL-NEXT: s_getpc_b64 s[12:13]
; GFX9-PAL-NEXT: s_mov_b32 s12, s0
; GFX9-PAL-NEXT: s_load_dwordx2 s[12:13], s[12:13], 0x0
-; GFX9-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-PAL-NEXT: s_mov_b32 s1, s0
-; GFX9-PAL-NEXT: s_mov_b32 s2, s0
-; GFX9-PAL-NEXT: s_mov_b32 s3, s0
+; GFX9-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-PAL-NEXT: s_and_b32 s13, s13, 0xffff
; GFX9-PAL-NEXT: s_add_u32 flat_scratch_lo, s12, s11
-; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-PAL-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, s2
; GFX9-PAL-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-PAL-NEXT: s_mov_b32 s0, 0
; GFX9-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:48
; GFX9-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:32
; GFX9-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:16
@@ -106,10 +98,8 @@ define amdgpu_kernel void @zero_init_kernel() {
;
; GFX942-LABEL: zero_init_kernel:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: s_mov_b32 s1, s0
-; GFX942-NEXT: s_mov_b32 s2, s0
-; GFX942-NEXT: s_mov_b32 s3, s0
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:48
@@ -129,14 +119,13 @@ define amdgpu_kernel void @zero_init_kernel() {
; GFX1010-PAL-NEXT: s_addc_u32 s13, s13, 0
; GFX1010-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX1010-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; GFX1010-PAL-NEXT: s_mov_b32 s0, 0
-; GFX1010-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1010-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1010-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1010-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1010-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1010-PAL-NEXT: v_mov_b32_e32 v2, s2
; GFX1010-PAL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1010-PAL-NEXT: s_mov_b32 s0, 0
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:48
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:32
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:16
@@ -154,10 +143,8 @@ define amdgpu_kernel void @zero_init_kernel() {
; GFX1030-PAL-NEXT: s_addc_u32 s13, s13, 0
; GFX1030-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX1030-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; GFX1030-PAL-NEXT: s_mov_b32 s0, 0
-; GFX1030-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1030-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1030-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1030-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1030-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1030-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -170,11 +157,9 @@ define amdgpu_kernel void @zero_init_kernel() {
;
; GFX11-PAL-LABEL: zero_init_kernel:
; GFX11-PAL: ; %bb.0:
-; GFX11-PAL-NEXT: s_mov_b32 s0, 0
+; GFX11-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-PAL-NEXT: s_mov_b32 s1, s0
-; GFX11-PAL-NEXT: s_mov_b32 s2, s0
-; GFX11-PAL-NEXT: s_mov_b32 s3, s0
+; GFX11-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-PAL-NEXT: s_clause 0x3
@@ -186,11 +171,9 @@ define amdgpu_kernel void @zero_init_kernel() {
;
; GFX12-PAL-LABEL: zero_init_kernel:
; GFX12-PAL: ; %bb.0:
-; GFX12-PAL-NEXT: s_mov_b32 s0, 0
+; GFX12-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX12-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-PAL-NEXT: s_mov_b32 s1, s0
-; GFX12-PAL-NEXT: s_mov_b32 s2, s0
-; GFX12-PAL-NEXT: s_mov_b32 s3, s0
+; GFX12-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-PAL-NEXT: s_clause 0x3
@@ -208,10 +191,8 @@ define void @zero_init_foo() {
; GFX9-LABEL: zero_init_foo:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_mov_b32 s1, s0
-; GFX9-NEXT: s_mov_b32 s2, s0
-; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -226,10 +207,8 @@ define void @zero_init_foo() {
; GFX10-LABEL: zero_init_foo:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -243,11 +222,9 @@ define void @zero_init_foo() {
; GFX11-LABEL: zero_init_foo:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_clause 0x3
@@ -264,13 +241,11 @@ define void @zero_init_foo() {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_alu 0xfffe
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: scratch_store_b128 off, v[0:3], s32 offset:48
@@ -282,10 +257,8 @@ define void @zero_init_foo() {
; GFX9-PAL-LABEL: zero_init_foo:
; GFX9-PAL: ; %bb.0:
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-PAL-NEXT: s_mov_b32 s1, s0
-; GFX9-PAL-NEXT: s_mov_b32 s2, s0
-; GFX9-PAL-NEXT: s_mov_b32 s3, s0
+; GFX9-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -300,10 +273,8 @@ define void @zero_init_foo() {
; GFX942-LABEL: zero_init_foo:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: s_mov_b32 s1, s0
-; GFX942-NEXT: s_mov_b32 s2, s0
-; GFX942-NEXT: s_mov_b32 s3, s0
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: scratch_store_dwordx4 off, v[0:3], s32 offset:48
@@ -316,10 +287,8 @@ define void @zero_init_foo() {
; GFX10-PAL-LABEL: zero_init_foo:
; GFX10-PAL: ; %bb.0:
; GFX10-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-PAL-NEXT: s_mov_b32 s0, 0
-; GFX10-PAL-NEXT: s_mov_b32 s1, s0
-; GFX10-PAL-NEXT: s_mov_b32 s2, s0
-; GFX10-PAL-NEXT: s_mov_b32 s3, s0
+; GFX10-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX10-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX10-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -333,11 +302,9 @@ define void @zero_init_foo() {
; GFX11-PAL-LABEL: zero_init_foo:
; GFX11-PAL: ; %bb.0:
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-PAL-NEXT: s_mov_b32 s0, 0
+; GFX11-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-PAL-NEXT: s_mov_b32 s1, s0
-; GFX11-PAL-NEXT: s_mov_b32 s2, s0
-; GFX11-PAL-NEXT: s_mov_b32 s3, s0
+; GFX11-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-PAL-NEXT: s_clause 0x3
@@ -354,13 +321,11 @@ define void @zero_init_foo() {
; GFX12-PAL-NEXT: s_wait_samplecnt 0x0
; GFX12-PAL-NEXT: s_wait_bvhcnt 0x0
; GFX12-PAL-NEXT: s_wait_kmcnt 0x0
-; GFX12-PAL-NEXT: s_mov_b32 s0, 0
-; GFX12-PAL-NEXT: s_wait_alu 0xfffe
-; GFX12-PAL-NEXT: s_mov_b32 s1, s0
-; GFX12-PAL-NEXT: s_mov_b32 s2, s0
-; GFX12-PAL-NEXT: s_mov_b32 s3, s0
+; GFX12-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX12-PAL-NEXT: s_wait_alu 0xfffe
+; GFX12-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-PAL-NEXT: s_wait_alu 0xfffe
; GFX12-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-PAL-NEXT: s_clause 0x3
; GFX12-PAL-NEXT: scratch_store_b128 off, v[0:3], s32 offset:48
@@ -1039,13 +1004,13 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX9-NEXT: s_mov_b32 s0, 0
; GFX9-NEXT: scratch_load_dword v0, off, s0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_mov_b32 s1, s0
-; GFX9-NEXT: s_mov_b32 s2, s0
-; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: s_mov_b32 s0, 0
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:272
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
@@ -1060,10 +1025,8 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
; GFX10-NEXT: scratch_load_dword v0, off, off glc dlc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -1078,11 +1041,9 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX11: ; %bb.0:
; GFX11-NEXT: scratch_load_b32 v0, off, off glc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_clause 0x3
@@ -1096,11 +1057,9 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX12: ; %bb.0:
; GFX12-NEXT: scratch_load_b32 v0, off, off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_clause 0x3
@@ -1116,19 +1075,19 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX9-PAL-NEXT: s_mov_b32 s12, s0
; GFX9-PAL-NEXT: s_load_dwordx2 s[12:13], s[12:13], 0x0
; GFX9-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-PAL-NEXT: s_mov_b32 s1, s0
-; GFX9-PAL-NEXT: s_mov_b32 s2, s0
-; GFX9-PAL-NEXT: s_mov_b32 s3, s0
; GFX9-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-PAL-NEXT: s_and_b32 s13, s13, 0xffff
; GFX9-PAL-NEXT: s_add_u32 flat_scratch_lo, s12, s11
; GFX9-PAL-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GFX9-PAL-NEXT: scratch_load_dword v0, off, s0 glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, s2
; GFX9-PAL-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-PAL-NEXT: s_mov_b32 s0, 0
; GFX9-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX9-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:272
; GFX9-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
@@ -1139,10 +1098,8 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX942: ; %bb.0:
; GFX942-NEXT: scratch_load_dword v0, off, off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: s_mov_b32 s1, s0
-; GFX942-NEXT: s_mov_b32 s2, s0
-; GFX942-NEXT: s_mov_b32 s3, s0
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:256
@@ -1165,13 +1122,14 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX1010-PAL-NEXT: s_mov_b32 s0, 0
; GFX1010-PAL-NEXT: scratch_load_dword v0, off, s0 glc dlc
; GFX1010-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX1010-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1010-PAL-NEXT: s_waitcnt_depctr 0xffe3
+; GFX1010-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1010-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1010-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1010-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1010-PAL-NEXT: v_mov_b32_e32 v2, s2
; GFX1010-PAL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1010-PAL-NEXT: s_mov_b32 s0, 0
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:256
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:272
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:288
@@ -1191,10 +1149,8 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX1030-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
; GFX1030-PAL-NEXT: scratch_load_dword v0, off, off glc dlc
; GFX1030-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-PAL-NEXT: s_mov_b32 s0, 0
-; GFX1030-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1030-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1030-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1030-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1030-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1030-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -1209,11 +1165,9 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX11-PAL: ; %bb.0:
; GFX11-PAL-NEXT: scratch_load_b32 v0, off, off glc dlc
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PAL-NEXT: s_mov_b32 s0, 0
+; GFX11-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-PAL-NEXT: s_mov_b32 s1, s0
-; GFX11-PAL-NEXT: s_mov_b32 s2, s0
-; GFX11-PAL-NEXT: s_mov_b32 s3, s0
+; GFX11-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-PAL-NEXT: s_clause 0x3
@@ -1227,11 +1181,9 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
; GFX12-PAL: ; %bb.0:
; GFX12-PAL-NEXT: scratch_load_b32 v0, off, off scope:SCOPE_SYS
; GFX12-PAL-NEXT: s_wait_loadcnt 0x0
-; GFX12-PAL-NEXT: s_mov_b32 s0, 0
+; GFX12-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX12-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-PAL-NEXT: s_mov_b32 s1, s0
-; GFX12-PAL-NEXT: s_mov_b32 s2, s0
-; GFX12-PAL-NEXT: s_mov_b32 s3, s0
+; GFX12-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-PAL-NEXT: s_clause 0x3
@@ -1254,10 +1206,8 @@ define void @zero_init_small_offset_foo() {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: scratch_load_dword v0, off, s32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_mov_b32 s1, s0
-; GFX9-NEXT: s_mov_b32 s2, s0
-; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -1274,10 +1224,8 @@ define void @zero_init_small_offset_foo() {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: scratch_load_dword v0, off, s32 glc dlc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -1293,11 +1241,9 @@ define void @zero_init_small_offset_foo() {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: scratch_load_b32 v0, off, s32 glc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_clause 0x3
@@ -1316,13 +1262,11 @@ define void @zero_init_small_offset_foo() {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: scratch_load_b32 v0, off, s32 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_alu 0xfffe
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: scratch_store_b128 off, v[0:3], s32 offset:256
@@ -1336,10 +1280,8 @@ define void @zero_init_small_offset_foo() {
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-PAL-NEXT: scratch_load_dword v0, off, s32 glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-PAL-NEXT: s_mov_b32 s1, s0
-; GFX9-PAL-NEXT: s_mov_b32 s2, s0
-; GFX9-PAL-NEXT: s_mov_b32 s3, s0
+; GFX9-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -1356,10 +1298,8 @@ define void @zero_init_small_offset_foo() {
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: scratch_load_dword v0, off, s32 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: s_mov_b32 s1, s0
-; GFX942-NEXT: s_mov_b32 s2, s0
-; GFX942-NEXT: s_mov_b32 s3, s0
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: scratch_store_dwordx4 off, v[0:3], s32 offset:256
@@ -1374,10 +1314,8 @@ define void @zero_init_small_offset_foo() {
; GFX10-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-PAL-NEXT: scratch_load_dword v0, off, s32 glc dlc
; GFX10-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-PAL-NEXT: s_mov_b32 s0, 0
-; GFX10-PAL-NEXT: s_mov_b32 s1, s0
-; GFX10-PAL-NEXT: s_mov_b32 s2, s0
-; GFX10-PAL-NEXT: s_mov_b32 s3, s0
+; GFX10-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX10-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX10-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -1393,11 +1331,9 @@ define void @zero_init_small_offset_foo() {
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-PAL-NEXT: scratch_load_b32 v0, off, s32 glc dlc
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PAL-NEXT: s_mov_b32 s0, 0
+; GFX11-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-PAL-NEXT: s_mov_b32 s1, s0
-; GFX11-PAL-NEXT: s_mov_b32 s2, s0
-; GFX11-PAL-NEXT: s_mov_b32 s3, s0
+; GFX11-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-PAL-NEXT: s_clause 0x3
@@ -1416,13 +1352,11 @@ define void @zero_init_small_offset_foo() {
; GFX12-PAL-NEXT: s_wait_kmcnt 0x0
; GFX12-PAL-NEXT: scratch_load_b32 v0, off, s32 scope:SCOPE_SYS
; GFX12-PAL-NEXT: s_wait_loadcnt 0x0
-; GFX12-PAL-NEXT: s_mov_b32 s0, 0
-; GFX12-PAL-NEXT: s_wait_alu 0xfffe
-; GFX12-PAL-NEXT: s_mov_b32 s1, s0
-; GFX12-PAL-NEXT: s_mov_b32 s2, s0
-; GFX12-PAL-NEXT: s_mov_b32 s3, s0
+; GFX12-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX12-PAL-NEXT: s_wait_alu 0xfffe
+; GFX12-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-PAL-NEXT: s_wait_alu 0xfffe
; GFX12-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-PAL-NEXT: s_clause 0x3
; GFX12-PAL-NEXT: scratch_store_b128 off, v[0:3], s32 offset:256
@@ -2233,9 +2167,8 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX9-NEXT: s_mov_b32 s0, 0
; GFX9-NEXT: scratch_load_dword v0, off, s0 offset:4 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_mov_b32 s1, s0
-; GFX9-NEXT: s_mov_b32 s2, s0
-; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -2255,10 +2188,8 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
; GFX10-NEXT: scratch_load_dword v0, off, off offset:4 glc dlc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -2274,11 +2205,9 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX11: ; %bb.0:
; GFX11-NEXT: scratch_load_b32 v0, off, off offset:4 glc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_movk_i32 s0, 0x4004
@@ -2293,11 +2222,9 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX12: ; %bb.0:
; GFX12-NEXT: scratch_load_b32 v0, off, off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_clause 0x3
@@ -2313,15 +2240,14 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX9-PAL-NEXT: s_mov_b32 s12, s0
; GFX9-PAL-NEXT: s_load_dwordx2 s[12:13], s[12:13], 0x0
; GFX9-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-PAL-NEXT: s_mov_b32 s1, s0
-; GFX9-PAL-NEXT: s_mov_b32 s2, s0
-; GFX9-PAL-NEXT: s_mov_b32 s3, s0
; GFX9-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-PAL-NEXT: s_and_b32 s13, s13, 0xffff
; GFX9-PAL-NEXT: s_add_u32 flat_scratch_lo, s12, s11
; GFX9-PAL-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GFX9-PAL-NEXT: scratch_load_dword v0, off, s0 offset:4 glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -2337,10 +2263,8 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX942: ; %bb.0:
; GFX942-NEXT: scratch_load_dword v0, off, off offset:4 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: s_mov_b32 s1, s0
-; GFX942-NEXT: s_mov_b32 s2, s0
-; GFX942-NEXT: s_mov_b32 s3, s0
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: s_movk_i32 s0, 0x4004
@@ -2364,9 +2288,9 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX1010-PAL-NEXT: s_mov_b32 s0, 0
; GFX1010-PAL-NEXT: scratch_load_dword v0, off, s0 offset:4 glc dlc
; GFX1010-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX1010-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1010-PAL-NEXT: s_waitcnt_depctr 0xffe3
+; GFX1010-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1010-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1010-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1010-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1010-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -2391,10 +2315,8 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX1030-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
; GFX1030-PAL-NEXT: scratch_load_dword v0, off, off offset:4 glc dlc
; GFX1030-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-PAL-NEXT: s_mov_b32 s0, 0
-; GFX1030-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1030-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1030-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1030-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1030-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1030-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -2410,11 +2332,9 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX11-PAL: ; %bb.0:
; GFX11-PAL-NEXT: scratch_load_b32 v0, off, off offset:4 glc dlc
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PAL-NEXT: s_mov_b32 s0, 0
+; GFX11-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-PAL-NEXT: s_mov_b32 s1, s0
-; GFX11-PAL-NEXT: s_mov_b32 s2, s0
-; GFX11-PAL-NEXT: s_mov_b32 s3, s0
+; GFX11-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-PAL-NEXT: s_movk_i32 s0, 0x4004
@@ -2429,11 +2349,9 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
; GFX12-PAL: ; %bb.0:
; GFX12-PAL-NEXT: scratch_load_b32 v0, off, off scope:SCOPE_SYS
; GFX12-PAL-NEXT: s_wait_loadcnt 0x0
-; GFX12-PAL-NEXT: s_mov_b32 s0, 0
+; GFX12-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX12-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-PAL-NEXT: s_mov_b32 s1, s0
-; GFX12-PAL-NEXT: s_mov_b32 s2, s0
-; GFX12-PAL-NEXT: s_mov_b32 s3, s0
+; GFX12-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-PAL-NEXT: s_clause 0x3
@@ -2456,10 +2374,8 @@ define void @zero_init_large_offset_foo() {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: scratch_load_dword v0, off, s32 offset:4 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_mov_b32 s0, 0
-; GFX9-NEXT: s_mov_b32 s1, s0
-; GFX9-NEXT: s_mov_b32 s2, s0
-; GFX9-NEXT: s_mov_b32 s3, s0
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -2480,10 +2396,8 @@ define void @zero_init_large_offset_foo() {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: scratch_load_dword v0, off, s32 offset:4 glc dlc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_mov_b32 s0, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -2503,11 +2417,9 @@ define void @zero_init_large_offset_foo() {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: scratch_load_b32 v0, off, s32 offset:4 glc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_add_i32 s0, s32, 0x4004
@@ -2529,13 +2441,11 @@ define void @zero_init_large_offset_foo() {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: scratch_load_b32 v0, off, s32 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_wait_alu 0xfffe
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: scratch_store_b128 off, v[0:3], s32 offset:16384
@@ -2549,10 +2459,8 @@ define void @zero_init_large_offset_foo() {
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-PAL-NEXT: scratch_load_dword v0, off, s32 offset:4 glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: s_mov_b32 s0, 0
-; GFX9-PAL-NEXT: s_mov_b32 s1, s0
-; GFX9-PAL-NEXT: s_mov_b32 s2, s0
-; GFX9-PAL-NEXT: s_mov_b32 s3, s0
+; GFX9-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -2573,10 +2481,8 @@ define void @zero_init_large_offset_foo() {
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: scratch_load_dword v0, off, s32 offset:4 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: s_mov_b32 s1, s0
-; GFX942-NEXT: s_mov_b32 s2, s0
-; GFX942-NEXT: s_mov_b32 s3, s0
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: s_add_i32 s0, s32, 0x4004
@@ -2595,10 +2501,8 @@ define void @zero_init_large_offset_foo() {
; GFX1010-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1010-PAL-NEXT: scratch_load_dword v0, off, s32 offset:4 glc dlc
; GFX1010-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX1010-PAL-NEXT: s_mov_b32 s0, 0
-; GFX1010-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1010-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1010-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1010-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1010-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1010-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1010-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -2621,10 +2525,8 @@ define void @zero_init_large_offset_foo() {
; GFX1030-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1030-PAL-NEXT: scratch_load_dword v0, off, s32 offset:4 glc dlc
; GFX1030-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-PAL-NEXT: s_mov_b32 s0, 0
-; GFX1030-PAL-NEXT: s_mov_b32 s1, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s2, s0
-; GFX1030-PAL-NEXT: s_mov_b32 s3, s0
+; GFX1030-PAL-NEXT: s_mov_b64 s[0:1], 0
+; GFX1030-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1030-PAL-NEXT: v_mov_b32_e32 v0, s0
; GFX1030-PAL-NEXT: v_mov_b32_e32 v1, s1
; GFX1030-PAL-NEXT: v_mov_b32_e32 v2, s2
@@ -2644,11 +2546,9 @@ define void @zero_init_large_offset_foo() {
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-PAL-NEXT: scratch_load_b32 v0, off, s32 offset:4 glc dlc
; GFX11-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PAL-NEXT: s_mov_b32 s0, 0
+; GFX11-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-PAL-NEXT: s_mov_b32 s1, s0
-; GFX11-PAL-NEXT: s_mov_b32 s2, s0
-; GFX11-PAL-NEXT: s_mov_b32 s3, s0
+; GFX11-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-PAL-NEXT: s_add_i32 s0, s32, 0x4004
@@ -2670,13 +2570,11 @@ define void @zero_init_large_offset_foo() {
; GFX12-PAL-NEXT: s_wait_kmcnt 0x0
; GFX12-PAL-NEXT: scratch_load_b32 v0, off, s32 scope:SCOPE_SYS
; GFX12-PAL-NEXT: s_wait_loadcnt 0x0
-; GFX12-PAL-NEXT: s_mov_b32 s0, 0
-; GFX12-PAL-NEXT: s_wait_alu 0xfffe
-; GFX12-PAL-NEXT: s_mov_b32 s1, s0
-; GFX12-PAL-NEXT: s_mov_b32 s2, s0
-; GFX12-PAL-NEXT: s_mov_b32 s3, s0
+; GFX12-PAL-NEXT: s_mov_b64 s[0:1], 0
; GFX12-PAL-NEXT: s_wait_alu 0xfffe
+; GFX12-PAL-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-PAL-NEXT: s_wait_alu 0xfffe
; GFX12-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-PAL-NEXT: s_clause 0x3
; GFX12-PAL-NEXT: scratch_store_b128 off, v[0:3], s32 offset:16384
@@ -4677,12 +4575,12 @@ bb:
define amdgpu_ps void @large_offset() {
; GFX9-LABEL: large_offset:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_add_u32 flat_scratch_lo, s0, s2
; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_add_u32 flat_scratch_lo, s0, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s1, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: s_mov_b32 s0, 0
; GFX9-NEXT: scratch_store_dwordx4 off, v[0:3], s0 offset:3024
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -4705,11 +4603,11 @@ define amdgpu_ps void @large_offset() {
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_movk_i32 s0, 0x810
; GFX10-NEXT: s_add_i32 s1, s0, 0x3c0
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: scratch_store_dwordx4 off, v[0:3], s1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: scratch_load_dwordx4 v[0:3], off, s1 glc dlc
@@ -4726,12 +4624,11 @@ define amdgpu_ps void @large_offset() {
; GFX11-LABEL: large_offset:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_mov_b32 s0, 16
; GFX11-NEXT: s_movk_i32 s1, 0x810
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: scratch_store_b128 off, v[0:3], off offset:3024 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: scratch_load_b128 v[0:3], off, off offset:3024 glc dlc
@@ -4747,11 +4644,11 @@ define amdgpu_ps void @large_offset() {
; GFX12-LABEL: large_offset:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: s_movk_i32 s1, 0x800
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: scratch_store_b128 off, v[0:3], off offset:3008 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: scratch_load_b128 v[0:3], off, off offset:3008 scope:SCOPE_SYS
@@ -4770,9 +4667,9 @@ define amdgpu_ps void @large_offset() {
; GFX9-PAL-NEXT: s_mov_b32 s2, s0
; GFX9-PAL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX9-PAL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-PAL-NEXT: v_mov_b32_e32 v3, v1
; GFX9-PAL-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v3, v0
; GFX9-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-PAL-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-PAL-NEXT: s_add_u32 flat_scratch_lo, s2, s0
@@ -4794,10 +4691,8 @@ define amdgpu_ps void @large_offset() {
;
; GFX942-LABEL: large_offset:
; GFX942: ; %bb.0: ; %bb
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], 0
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:3024 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: scratch_load_dwordx4 v[0:3], off, off offset:3024 sc0 sc1
@@ -4824,11 +4719,11 @@ define amdgpu_ps void @large_offset() {
; GFX1010-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
; GFX1010-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
; GFX1010-PAL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1010-PAL-NEXT: v_mov_b32_e32 v1, 0
; GFX1010-PAL-NEXT: s_movk_i32 s0, 0x810
; GFX1010-PAL-NEXT: s_add_i32 s1, s0, 0x3c0
-; GFX1010-PAL-NEXT: v_mov_b32_e32 v1, v0
+; GFX1010-PAL-NEXT: v_mov_b32_e32 v3, v1
; GFX1010-PAL-NEXT: v_mov_b32_e32 v2, v0
-; GFX1010-PAL-NEXT: v_mov_b32_e32 v3, v0
; GFX1010-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s1
; GFX1010-PAL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010-PAL-NEXT: scratch_load_dwordx4 v[0:3], off, s1 glc dlc
@@ -4855,11 +4750,11 @@ define amdgpu_ps void @large_offset() {
; GFX1030-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
; GFX1030-PAL-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
; GFX1030-PAL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1030-PAL-NEXT: v_mov_b32_e32 v1, 0
; GFX1030-PAL-NEXT: s_movk_i32 s0, 0x810
; GFX1030-PAL-NEXT: s_add_i32 s1, s0, 0x3c0
-; GFX1030-PAL-NEXT: v_mov_b32_e32 v1, v0
+; GFX1030-PAL-NEXT: v_mov_b32_e32 v3, v1
; GFX1030-PAL-NEXT: v_mov_b32_e32 v2, v0
-; GFX1030-PAL-NEXT: v_mov_b32_e32 v3, v0
; GFX1030-PAL-NEXT: scratch_store_dwordx4 off, v[0:3], s1
; GFX1030-PAL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1030-PAL-NEXT: scratch_load_dwordx4 v[0:3], off, s1 glc dlc
@@ -4876,12 +4771,11 @@ define amdgpu_ps void @large_offset() {
; GFX11-PAL-LABEL: large_offset:
; GFX11-PAL: ; %bb.0: ; %bb
; GFX11-PAL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-PAL-NEXT: v_mov_b32_e32 v1, 0
; GFX11-PAL-NEXT: s_mov_b32 s0, 16
; GFX11-PAL-NEXT: s_movk_i32 s1, 0x810
; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-PAL-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-PAL-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-PAL-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-PAL-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], off offset:3024 dlc
; GFX11-PAL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-PAL-NEXT: scratch_load_b128 v[0:3], off, off offset:3024 glc dlc
@@ -4897,11 +4791,11 @@ define amdgpu_ps void @large_offset() {
; GFX12-PAL-LABEL: large_offset:
; GFX12-PAL: ; %bb.0: ; %bb
; GFX12-PAL-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-PAL-NEXT: v_mov_b32_e32 v1, 0
; GFX12-PAL-NEXT: s_mov_b32 s0, 0
; GFX12-PAL-NEXT: s_movk_i32 s1, 0x800
; GFX12-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-PAL-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; GFX12-PAL-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-PAL-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-PAL-NEXT: scratch_store_b128 off, v[0:3], off offset:3008 scope:SCOPE_SYS
; GFX12-PAL-NEXT: s_wait_storecnt 0x0
; GFX12-PAL-NEXT: scratch_load_b128 v[0:3], off, off offset:3008 scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index 6384fdba7a45a..232e394c5fc2d 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -1982,11 +1982,9 @@ define amdgpu_gfx <512 x i32> @return_512xi32() #0 {
; GFX11-LABEL: return_512xi32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s3, s0
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
; GFX11-NEXT: s_clause 0x1f
@@ -3186,11 +3184,9 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-NEXT: scratch_store_b32 off, v60, s33 offset:1600 ; 4-byte Folded Spill
; GFX11-NEXT: s_mov_b32 exec_lo, s0
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
; GFX11-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_mov_b32 s36, s34
diff --git a/llvm/test/CodeGen/AMDGPU/hazard-recognizer-src-shared-base.ll b/llvm/test/CodeGen/AMDGPU/hazard-recognizer-src-shared-base.ll
index 4aa49f2c9296d..11f931aacbc96 100644
--- a/llvm/test/CodeGen/AMDGPU/hazard-recognizer-src-shared-base.ll
+++ b/llvm/test/CodeGen/AMDGPU/hazard-recognizer-src-shared-base.ll
@@ -5,9 +5,9 @@ define amdgpu_kernel void @foo() {
; CHECK-LABEL: foo:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_mov_b64 s[0:1], src_shared_base
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s1
-; CHECK-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v0
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
+; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 0
; CHECK-NEXT: flat_store_b64 v[0:1], v[2:3]
; CHECK-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
index 5dff660912e40..9d47adccdfccd 100644
--- a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
@@ -6,160 +6,111 @@ define void @main(i1 %arg) #0 {
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
-; CHECK-NEXT: v_writelane_b32 v5, s30, 0
-; CHECK-NEXT: v_writelane_b32 v5, s31, 1
-; CHECK-NEXT: v_writelane_b32 v5, s36, 2
-; CHECK-NEXT: v_writelane_b32 v5, s37, 3
-; CHECK-NEXT: v_writelane_b32 v5, s38, 4
-; CHECK-NEXT: v_writelane_b32 v5, s39, 5
-; CHECK-NEXT: v_writelane_b32 v5, s40, 6
-; CHECK-NEXT: v_writelane_b32 v5, s41, 7
-; CHECK-NEXT: v_writelane_b32 v5, s42, 8
-; CHECK-NEXT: v_writelane_b32 v5, s43, 9
-; CHECK-NEXT: v_writelane_b32 v5, s44, 10
-; CHECK-NEXT: v_writelane_b32 v5, s45, 11
-; CHECK-NEXT: v_writelane_b32 v5, s46, 12
-; CHECK-NEXT: v_writelane_b32 v5, s47, 13
-; CHECK-NEXT: v_writelane_b32 v5, s48, 14
-; CHECK-NEXT: v_writelane_b32 v5, s49, 15
-; CHECK-NEXT: s_getpc_b64 s[24:25]
-; CHECK-NEXT: v_writelane_b32 v5, s50, 16
-; CHECK-NEXT: s_movk_i32 s4, 0xf0
-; CHECK-NEXT: s_mov_b32 s5, s24
-; CHECK-NEXT: v_writelane_b32 v5, s51, 17
-; CHECK-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
-; CHECK-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane
-; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: s_load_dwordx4 s[28:31], s[4:5], 0x0
-; CHECK-NEXT: s_movk_i32 s20, 0x130
-; CHECK-NEXT: s_mov_b32 s21, s24
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_writelane_b32 v7, s36, 0
-; CHECK-NEXT: v_writelane_b32 v7, s37, 1
-; CHECK-NEXT: v_writelane_b32 v7, s38, 2
-; CHECK-NEXT: v_writelane_b32 v7, s39, 3
-; CHECK-NEXT: v_writelane_b32 v7, s40, 4
-; CHECK-NEXT: v_writelane_b32 v7, s41, 5
-; CHECK-NEXT: v_writelane_b32 v7, s42, 6
-; CHECK-NEXT: v_writelane_b32 v7, s43, 7
-; CHECK-NEXT: v_writelane_b32 v7, s44, 8
-; CHECK-NEXT: v_writelane_b32 v7, s45, 9
-; CHECK-NEXT: v_writelane_b32 v7, s46, 10
-; CHECK-NEXT: s_load_dwordx16 s[4:19], s[20:21], 0x0
-; CHECK-NEXT: v_writelane_b32 v7, s47, 11
-; CHECK-NEXT: v_writelane_b32 v7, s48, 12
-; CHECK-NEXT: s_mov_b32 s20, 0
-; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: v_writelane_b32 v7, s49, 13
-; CHECK-NEXT: v_mov_b32_e32 v2, s28
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
-; CHECK-NEXT: s_mov_b32 s21, s20
-; CHECK-NEXT: s_mov_b32 s22, s20
-; CHECK-NEXT: s_mov_b32 s23, s20
-; CHECK-NEXT: v_writelane_b32 v7, s50, 14
-; CHECK-NEXT: v_writelane_b32 v7, s51, 15
-; CHECK-NEXT: image_sample_lz v3, v[2:3], s[44:51], s[20:23] dmask:0x1
-; CHECK-NEXT: v_mov_b32_e32 v2, v1
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_writelane_b32 v7, s4, 16
-; CHECK-NEXT: v_writelane_b32 v7, s5, 17
-; CHECK-NEXT: v_writelane_b32 v7, s6, 18
-; CHECK-NEXT: v_writelane_b32 v7, s7, 19
-; CHECK-NEXT: v_writelane_b32 v7, s8, 20
-; CHECK-NEXT: v_writelane_b32 v7, s9, 21
-; CHECK-NEXT: image_sample_lz v4, v[1:2], s[4:11], s[20:23] dmask:0x1
-; CHECK-NEXT: v_writelane_b32 v7, s10, 22
-; CHECK-NEXT: v_writelane_b32 v7, s11, 23
-; CHECK-NEXT: v_writelane_b32 v7, s12, 24
-; CHECK-NEXT: v_writelane_b32 v7, s13, 25
-; CHECK-NEXT: v_writelane_b32 v7, s14, 26
-; CHECK-NEXT: v_writelane_b32 v7, s15, 27
-; CHECK-NEXT: v_writelane_b32 v5, s52, 18
-; CHECK-NEXT: v_writelane_b32 v7, s16, 28
-; CHECK-NEXT: v_writelane_b32 v5, s53, 19
-; CHECK-NEXT: v_writelane_b32 v7, s17, 29
-; CHECK-NEXT: v_writelane_b32 v5, s54, 20
-; CHECK-NEXT: v_writelane_b32 v7, s18, 30
-; CHECK-NEXT: s_mov_b32 s26, 48
-; CHECK-NEXT: s_mov_b32 s27, s24
-; CHECK-NEXT: v_writelane_b32 v5, s55, 21
-; CHECK-NEXT: v_writelane_b32 v7, s19, 31
-; CHECK-NEXT: s_load_dwordx8 s[4:11], s[26:27], 0x0
-; CHECK-NEXT: v_writelane_b32 v5, s56, 22
-; CHECK-NEXT: v_writelane_b32 v5, s57, 23
-; CHECK-NEXT: v_writelane_b32 v5, s58, 24
-; CHECK-NEXT: v_writelane_b32 v5, s59, 25
-; CHECK-NEXT: v_writelane_b32 v5, s60, 26
+; CHECK-NEXT: v_writelane_b32 v6, s30, 0
+; CHECK-NEXT: v_writelane_b32 v6, s31, 1
+; CHECK-NEXT: v_writelane_b32 v6, s34, 2
+; CHECK-NEXT: v_writelane_b32 v6, s35, 3
+; CHECK-NEXT: v_writelane_b32 v6, s36, 4
+; CHECK-NEXT: v_writelane_b32 v6, s37, 5
+; CHECK-NEXT: v_writelane_b32 v6, s38, 6
+; CHECK-NEXT: v_writelane_b32 v6, s39, 7
+; CHECK-NEXT: v_writelane_b32 v6, s40, 8
+; CHECK-NEXT: v_writelane_b32 v6, s41, 9
+; CHECK-NEXT: v_writelane_b32 v6, s42, 10
+; CHECK-NEXT: v_writelane_b32 v6, s43, 11
+; CHECK-NEXT: v_writelane_b32 v6, s44, 12
+; CHECK-NEXT: v_writelane_b32 v6, s45, 13
+; CHECK-NEXT: v_writelane_b32 v6, s46, 14
+; CHECK-NEXT: v_writelane_b32 v6, s47, 15
+; CHECK-NEXT: v_writelane_b32 v6, s48, 16
+; CHECK-NEXT: v_writelane_b32 v6, s49, 17
+; CHECK-NEXT: v_writelane_b32 v6, s50, 18
+; CHECK-NEXT: v_writelane_b32 v6, s51, 19
+; CHECK-NEXT: v_writelane_b32 v6, s52, 20
+; CHECK-NEXT: v_writelane_b32 v6, s53, 21
+; CHECK-NEXT: v_writelane_b32 v6, s54, 22
+; CHECK-NEXT: v_writelane_b32 v6, s55, 23
+; CHECK-NEXT: v_writelane_b32 v6, s56, 24
+; CHECK-NEXT: v_writelane_b32 v6, s57, 25
+; CHECK-NEXT: v_writelane_b32 v6, s58, 26
+; CHECK-NEXT: v_writelane_b32 v6, s59, 27
+; CHECK-NEXT: v_writelane_b32 v6, s60, 28
+; CHECK-NEXT: v_writelane_b32 v6, s61, 29
+; CHECK-NEXT: v_writelane_b32 v6, s62, 30
+; CHECK-NEXT: v_writelane_b32 v6, s63, 31
+; CHECK-NEXT: v_writelane_b32 v6, s64, 32
+; CHECK-NEXT: v_writelane_b32 v6, s65, 33
+; CHECK-NEXT: v_writelane_b32 v6, s66, 34
+; CHECK-NEXT: s_getpc_b64 s[4:5]
+; CHECK-NEXT: s_movk_i32 s6, 0xf0
+; CHECK-NEXT: s_mov_b64 s[12:13], 0
+; CHECK-NEXT: v_writelane_b32 v6, s67, 35
+; CHECK-NEXT: s_mov_b32 s7, s4
+; CHECK-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
+; CHECK-NEXT: s_load_dwordx16 s[52:67], s[6:7], 0x0
+; CHECK-NEXT: s_movk_i32 s6, 0x130
+; CHECK-NEXT: s_load_dwordx16 s[12:27], s[6:7], 0x0
+; CHECK-NEXT: s_mov_b64 s[28:29], 0
+; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_writelane_b32 v7, s4, 32
-; CHECK-NEXT: v_writelane_b32 v5, s61, 27
-; CHECK-NEXT: v_writelane_b32 v7, s5, 33
-; CHECK-NEXT: v_writelane_b32 v5, s62, 28
-; CHECK-NEXT: v_writelane_b32 v7, s6, 34
-; CHECK-NEXT: v_writelane_b32 v5, s63, 29
-; CHECK-NEXT: v_writelane_b32 v7, s7, 35
-; CHECK-NEXT: v_writelane_b32 v5, s64, 30
-; CHECK-NEXT: v_writelane_b32 v7, s8, 36
-; CHECK-NEXT: v_writelane_b32 v5, s65, 31
-; CHECK-NEXT: v_writelane_b32 v7, s9, 37
-; CHECK-NEXT: v_writelane_b32 v5, s66, 32
-; CHECK-NEXT: s_movk_i32 s28, 0x1f0
-; CHECK-NEXT: s_movk_i32 s30, 0x2f0
-; CHECK-NEXT: s_mov_b32 s29, s24
-; CHECK-NEXT: s_mov_b32 s31, s24
-; CHECK-NEXT: v_writelane_b32 v7, s10, 38
-; CHECK-NEXT: v_writelane_b32 v5, s67, 33
-; CHECK-NEXT: v_writelane_b32 v7, s11, 39
-; CHECK-NEXT: s_load_dwordx16 s[52:67], s[28:29], 0x0
-; CHECK-NEXT: s_load_dwordx16 s[4:19], s[30:31], 0x0
+; CHECK-NEXT: v_mov_b32_e32 v1, s8
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: s_mov_b64 s[30:31], s[28:29]
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane
+; CHECK-NEXT: image_sample_lz v1, v[1:2], s[60:67], s[28:31] dmask:0x1
+; CHECK-NEXT: v_writelane_b32 v7, s12, 0
+; CHECK-NEXT: v_writelane_b32 v7, s13, 1
+; CHECK-NEXT: v_writelane_b32 v7, s14, 2
+; CHECK-NEXT: v_writelane_b32 v7, s15, 3
+; CHECK-NEXT: v_writelane_b32 v7, s16, 4
+; CHECK-NEXT: v_writelane_b32 v7, s17, 5
+; CHECK-NEXT: image_sample_lz v5, v[3:4], s[12:19], s[28:31] dmask:0x1
+; CHECK-NEXT: v_writelane_b32 v7, s18, 6
+; CHECK-NEXT: v_writelane_b32 v7, s19, 7
+; CHECK-NEXT: v_writelane_b32 v7, s20, 8
+; CHECK-NEXT: v_writelane_b32 v7, s21, 9
+; CHECK-NEXT: v_writelane_b32 v7, s22, 10
+; CHECK-NEXT: v_writelane_b32 v7, s23, 11
+; CHECK-NEXT: v_writelane_b32 v7, s24, 12
+; CHECK-NEXT: v_writelane_b32 v6, s68, 36
+; CHECK-NEXT: v_writelane_b32 v7, s25, 13
+; CHECK-NEXT: v_writelane_b32 v6, s69, 37
+; CHECK-NEXT: v_writelane_b32 v7, s26, 14
+; CHECK-NEXT: s_mov_b32 s6, 48
+; CHECK-NEXT: s_movk_i32 s68, 0x2f0
+; CHECK-NEXT: s_mov_b32 s69, s4
+; CHECK-NEXT: v_writelane_b32 v7, s27, 15
+; CHECK-NEXT: s_movk_i32 s34, 0x1f0
+; CHECK-NEXT: s_mov_b32 s35, s4
+; CHECK-NEXT: s_load_dwordx8 s[20:27], s[6:7], 0x0
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0
+; CHECK-NEXT: s_load_dwordx16 s[36:51], s[68:69], 0x0
; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; CHECK-NEXT: s_xor_b64 s[24:25], vcc, -1
+; CHECK-NEXT: v_writelane_b32 v6, s70, 38
+; CHECK-NEXT: s_xor_b64 s[34:35], vcc, -1
+; CHECK-NEXT: v_writelane_b32 v6, s71, 39
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mul_f32_e32 v0, v4, v3
-; CHECK-NEXT: s_and_saveexec_b64 s[26:27], s[24:25]
-; CHECK-NEXT: s_xor_b64 s[26:27], exec, s[26:27]
+; CHECK-NEXT: v_mul_f32_e32 v0, v5, v1
+; CHECK-NEXT: s_and_saveexec_b64 vcc, s[34:35]
+; CHECK-NEXT: s_xor_b64 s[68:69], exec, vcc
; CHECK-NEXT: s_cbranch_execz .LBB0_3
; CHECK-NEXT: ; %bb.1: ; %bb48
-; CHECK-NEXT: v_readlane_b32 s36, v7, 0
-; CHECK-NEXT: v_readlane_b32 s44, v7, 8
-; CHECK-NEXT: v_readlane_b32 s45, v7, 9
-; CHECK-NEXT: v_readlane_b32 s46, v7, 10
-; CHECK-NEXT: v_readlane_b32 s47, v7, 11
-; CHECK-NEXT: v_readlane_b32 s48, v7, 12
-; CHECK-NEXT: v_readlane_b32 s49, v7, 13
-; CHECK-NEXT: v_readlane_b32 s50, v7, 14
-; CHECK-NEXT: v_readlane_b32 s51, v7, 15
+; CHECK-NEXT: image_sample_lz v3, v[3:4], s[60:67], s[28:31] dmask:0x1
+; CHECK-NEXT: v_mov_b32_e32 v1, v2
; CHECK-NEXT: s_and_b64 vcc, exec, -1
-; CHECK-NEXT: v_readlane_b32 s37, v7, 1
-; CHECK-NEXT: v_readlane_b32 s38, v7, 2
-; CHECK-NEXT: v_readlane_b32 s39, v7, 3
-; CHECK-NEXT: v_readlane_b32 s40, v7, 4
-; CHECK-NEXT: image_sample_lz v3, v[1:2], s[44:51], s[20:23] dmask:0x1
-; CHECK-NEXT: v_mov_b32_e32 v2, 0
-; CHECK-NEXT: v_readlane_b32 s41, v7, 5
-; CHECK-NEXT: v_readlane_b32 s42, v7, 6
-; CHECK-NEXT: v_readlane_b32 s43, v7, 7
; CHECK-NEXT: .LBB0_2: ; %bb50
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_readlane_b32 s36, v7, 32
-; CHECK-NEXT: v_readlane_b32 s40, v7, 36
-; CHECK-NEXT: v_readlane_b32 s41, v7, 37
-; CHECK-NEXT: v_readlane_b32 s42, v7, 38
-; CHECK-NEXT: v_readlane_b32 s43, v7, 39
-; CHECK-NEXT: s_mov_b32 s21, s20
-; CHECK-NEXT: s_mov_b32 s22, s20
-; CHECK-NEXT: s_mov_b32 s23, s20
-; CHECK-NEXT: v_readlane_b32 s37, v7, 33
-; CHECK-NEXT: v_readlane_b32 s38, v7, 34
+; CHECK-NEXT: s_mov_b64 s[30:31], s[28:29]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: image_sample_lz v4, v[1:2], s[60:67], s[40:43] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s39, v7, 35
-; CHECK-NEXT: image_sample_lz v1, v[1:2], s[12:19], s[20:23] dmask:0x1
+; CHECK-NEXT: image_sample_lz v4, v[1:2], s[12:19], s[24:27] dmask:0x1
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: image_sample_lz v1, v[1:2], s[44:51], s[28:31] dmask:0x1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_sub_f32_e32 v1, v1, v4
; CHECK-NEXT: v_mul_f32_e32 v1, v1, v0
@@ -168,158 +119,53 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_cbranch_vccnz .LBB0_2
; CHECK-NEXT: .LBB0_3: ; %Flow14
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_readlane_b32 s12, v7, 32
-; CHECK-NEXT: v_readlane_b32 s13, v7, 33
-; CHECK-NEXT: v_readlane_b32 s14, v7, 34
-; CHECK-NEXT: v_readlane_b32 s15, v7, 35
-; CHECK-NEXT: v_readlane_b32 s16, v7, 36
-; CHECK-NEXT: v_readlane_b32 s17, v7, 37
-; CHECK-NEXT: v_readlane_b32 s18, v7, 38
-; CHECK-NEXT: v_readlane_b32 s19, v7, 39
-; CHECK-NEXT: v_writelane_b32 v7, s4, 40
-; CHECK-NEXT: v_writelane_b32 v7, s5, 41
-; CHECK-NEXT: v_writelane_b32 v7, s6, 42
-; CHECK-NEXT: v_writelane_b32 v7, s7, 43
-; CHECK-NEXT: v_writelane_b32 v7, s8, 44
-; CHECK-NEXT: v_writelane_b32 v7, s9, 45
-; CHECK-NEXT: v_writelane_b32 v7, s10, 46
-; CHECK-NEXT: v_writelane_b32 v7, s11, 47
-; CHECK-NEXT: v_writelane_b32 v7, s12, 48
-; CHECK-NEXT: v_writelane_b32 v7, s13, 49
-; CHECK-NEXT: v_writelane_b32 v7, s14, 50
-; CHECK-NEXT: v_writelane_b32 v7, s15, 51
-; CHECK-NEXT: v_writelane_b32 v7, s16, 52
-; CHECK-NEXT: v_writelane_b32 v7, s17, 53
-; CHECK-NEXT: v_writelane_b32 v7, s18, 54
-; CHECK-NEXT: v_writelane_b32 v7, s19, 55
-; CHECK-NEXT: ; implicit-def: $vgpr6 : SGPR spill to VGPR lane
-; CHECK-NEXT: v_writelane_b32 v7, s52, 56
-; CHECK-NEXT: v_writelane_b32 v6, s60, 0
-; CHECK-NEXT: v_writelane_b32 v7, s53, 57
-; CHECK-NEXT: v_writelane_b32 v6, s61, 1
-; CHECK-NEXT: v_writelane_b32 v7, s54, 58
-; CHECK-NEXT: v_writelane_b32 v6, s62, 2
-; CHECK-NEXT: v_writelane_b32 v7, s55, 59
-; CHECK-NEXT: v_writelane_b32 v6, s63, 3
-; CHECK-NEXT: v_writelane_b32 v7, s56, 60
-; CHECK-NEXT: v_writelane_b32 v6, s64, 4
-; CHECK-NEXT: v_writelane_b32 v7, s57, 61
-; CHECK-NEXT: v_writelane_b32 v6, s65, 5
-; CHECK-NEXT: v_writelane_b32 v7, s58, 62
-; CHECK-NEXT: v_writelane_b32 v6, s66, 6
-; CHECK-NEXT: v_writelane_b32 v7, s59, 63
-; CHECK-NEXT: v_writelane_b32 v6, s67, 7
-; CHECK-NEXT: s_andn2_saveexec_b64 s[20:21], s[26:27]
+; CHECK-NEXT: s_andn2_saveexec_b64 s[12:13], s[68:69]
; CHECK-NEXT: s_cbranch_execz .LBB0_10
; CHECK-NEXT: ; %bb.4: ; %bb32
-; CHECK-NEXT: s_and_saveexec_b64 s[8:9], s[24:25]
-; CHECK-NEXT: s_xor_b64 s[22:23], exec, s[8:9]
+; CHECK-NEXT: s_and_saveexec_b64 s[14:15], s[34:35]
+; CHECK-NEXT: s_xor_b64 s[14:15], exec, s[14:15]
; CHECK-NEXT: s_cbranch_execz .LBB0_6
; CHECK-NEXT: ; %bb.5: ; %bb43
-; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: s_mov_b32 s9, s8
-; CHECK-NEXT: v_mov_b32_e32 v0, s8
-; CHECK-NEXT: v_readlane_b32 s36, v7, 0
-; CHECK-NEXT: v_mov_b32_e32 v1, s9
-; CHECK-NEXT: s_mov_b32 s10, s8
-; CHECK-NEXT: s_mov_b32 s11, s8
-; CHECK-NEXT: v_readlane_b32 s37, v7, 1
-; CHECK-NEXT: v_readlane_b32 s38, v7, 2
-; CHECK-NEXT: v_readlane_b32 s39, v7, 3
-; CHECK-NEXT: v_readlane_b32 s40, v7, 4
-; CHECK-NEXT: v_readlane_b32 s41, v7, 5
-; CHECK-NEXT: v_readlane_b32 s42, v7, 6
-; CHECK-NEXT: v_readlane_b32 s43, v7, 7
-; CHECK-NEXT: v_readlane_b32 s44, v7, 8
-; CHECK-NEXT: v_readlane_b32 s45, v7, 9
-; CHECK-NEXT: v_readlane_b32 s46, v7, 10
-; CHECK-NEXT: v_readlane_b32 s47, v7, 11
-; CHECK-NEXT: v_readlane_b32 s48, v7, 12
-; CHECK-NEXT: v_readlane_b32 s49, v7, 13
-; CHECK-NEXT: v_readlane_b32 s50, v7, 14
-; CHECK-NEXT: v_readlane_b32 s51, v7, 15
-; CHECK-NEXT: image_sample_lz v2, v[0:1], s[36:43], s[8:11] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s36, v7, 16
-; CHECK-NEXT: v_readlane_b32 s44, v7, 24
-; CHECK-NEXT: v_readlane_b32 s45, v7, 25
-; CHECK-NEXT: v_readlane_b32 s46, v7, 26
-; CHECK-NEXT: v_readlane_b32 s47, v7, 27
-; CHECK-NEXT: v_readlane_b32 s48, v7, 28
-; CHECK-NEXT: v_readlane_b32 s49, v7, 29
-; CHECK-NEXT: v_readlane_b32 s50, v7, 30
-; CHECK-NEXT: v_readlane_b32 s51, v7, 31
+; CHECK-NEXT: s_mov_b64 s[16:17], 0
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, v3
-; CHECK-NEXT: v_readlane_b32 s37, v7, 17
-; CHECK-NEXT: v_readlane_b32 s38, v7, 18
-; CHECK-NEXT: v_readlane_b32 s39, v7, 19
-; CHECK-NEXT: image_sample_lz v0, v[0:1], s[44:51], s[12:15] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s40, v7, 20
-; CHECK-NEXT: v_readlane_b32 s41, v7, 21
-; CHECK-NEXT: v_readlane_b32 s42, v7, 22
-; CHECK-NEXT: v_readlane_b32 s43, v7, 23
+; CHECK-NEXT: s_mov_b64 s[18:19], s[16:17]
+; CHECK-NEXT: image_sample_lz v1, v[2:3], s[52:59], s[16:19] dmask:0x1
+; CHECK-NEXT: v_readlane_b32 s44, v7, 0
+; CHECK-NEXT: v_readlane_b32 s52, v7, 8
+; CHECK-NEXT: v_readlane_b32 s53, v7, 9
+; CHECK-NEXT: v_readlane_b32 s54, v7, 10
+; CHECK-NEXT: v_readlane_b32 s55, v7, 11
+; CHECK-NEXT: v_readlane_b32 s56, v7, 12
+; CHECK-NEXT: v_readlane_b32 s57, v7, 13
+; CHECK-NEXT: v_readlane_b32 s58, v7, 14
+; CHECK-NEXT: v_readlane_b32 s59, v7, 15
+; CHECK-NEXT: v_readlane_b32 s45, v7, 1
+; CHECK-NEXT: v_readlane_b32 s46, v7, 2
+; CHECK-NEXT: v_readlane_b32 s47, v7, 3
+; CHECK-NEXT: v_readlane_b32 s48, v7, 4
+; CHECK-NEXT: v_readlane_b32 s49, v7, 5
+; CHECK-NEXT: image_sample_lz v0, v[2:3], s[52:59], s[20:23] dmask:0x1
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
+; CHECK-NEXT: v_readlane_b32 s50, v7, 6
+; CHECK-NEXT: v_readlane_b32 s51, v7, 7
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx3 v[2:4], off, s[8:11], 0
+; CHECK-NEXT: buffer_store_dwordx3 v[1:3], off, s[16:19], 0
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: .LBB0_6: ; %Flow12
-; CHECK-NEXT: s_or_saveexec_b64 s[4:5], s[22:23]
-; CHECK-NEXT: v_readlane_b32 s52, v7, 40
-; CHECK-NEXT: v_readlane_b32 s53, v7, 41
-; CHECK-NEXT: v_readlane_b32 s54, v7, 42
-; CHECK-NEXT: v_readlane_b32 s55, v7, 43
-; CHECK-NEXT: v_readlane_b32 s56, v7, 44
-; CHECK-NEXT: v_readlane_b32 s57, v7, 45
-; CHECK-NEXT: v_readlane_b32 s58, v7, 46
-; CHECK-NEXT: v_readlane_b32 s59, v7, 47
-; CHECK-NEXT: v_readlane_b32 s60, v7, 48
-; CHECK-NEXT: v_readlane_b32 s61, v7, 49
-; CHECK-NEXT: v_readlane_b32 s62, v7, 50
-; CHECK-NEXT: v_readlane_b32 s63, v7, 51
-; CHECK-NEXT: v_readlane_b32 s64, v7, 52
-; CHECK-NEXT: v_readlane_b32 s65, v7, 53
-; CHECK-NEXT: v_readlane_b32 s66, v7, 54
-; CHECK-NEXT: v_readlane_b32 s67, v7, 55
-; CHECK-NEXT: s_xor_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_andn2_saveexec_b64 s[14:15], s[14:15]
; CHECK-NEXT: s_cbranch_execz .LBB0_9
; CHECK-NEXT: ; %bb.7: ; %bb33.preheader
-; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: s_mov_b32 s6, s8
-; CHECK-NEXT: s_mov_b32 s7, s8
-; CHECK-NEXT: v_mov_b32_e32 v1, s6
-; CHECK-NEXT: v_readlane_b32 s36, v7, 56
-; CHECK-NEXT: s_mov_b32 s9, s8
-; CHECK-NEXT: s_mov_b32 s10, s8
-; CHECK-NEXT: s_mov_b32 s11, s8
-; CHECK-NEXT: v_mov_b32_e32 v2, s7
-; CHECK-NEXT: v_readlane_b32 s37, v7, 57
-; CHECK-NEXT: v_readlane_b32 s38, v7, 58
-; CHECK-NEXT: v_readlane_b32 s39, v7, 59
-; CHECK-NEXT: v_readlane_b32 s40, v7, 60
-; CHECK-NEXT: v_readlane_b32 s41, v7, 61
-; CHECK-NEXT: v_readlane_b32 s42, v7, 62
-; CHECK-NEXT: v_readlane_b32 s43, v7, 63
-; CHECK-NEXT: s_nop 4
-; CHECK-NEXT: image_sample_lz v3, v[1:2], s[36:43], s[8:11] dmask:0x1
-; CHECK-NEXT: image_sample_lz v4, v[1:2], s[52:59], s[8:11] dmask:0x1
-; CHECK-NEXT: ; kill: killed $vgpr1_vgpr2
-; CHECK-NEXT: s_mov_b64 s[12:13], s[36:37]
+; CHECK-NEXT: s_mov_b64 s[16:17], 0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: s_mov_b64 s[18:19], s[16:17]
+; CHECK-NEXT: image_sample_lz v3, v[1:2], s[4:11], s[16:19] dmask:0x1
+; CHECK-NEXT: image_sample_lz v4, v[1:2], s[36:43], s[16:19] dmask:0x1
; CHECK-NEXT: s_and_b64 vcc, exec, 0
-; CHECK-NEXT: v_readlane_b32 s44, v6, 0
-; CHECK-NEXT: v_readlane_b32 s45, v6, 1
-; CHECK-NEXT: v_readlane_b32 s46, v6, 2
-; CHECK-NEXT: v_readlane_b32 s47, v6, 3
-; CHECK-NEXT: v_readlane_b32 s48, v6, 4
-; CHECK-NEXT: v_readlane_b32 s49, v6, 5
-; CHECK-NEXT: v_readlane_b32 s50, v6, 6
-; CHECK-NEXT: v_readlane_b32 s51, v6, 7
-; CHECK-NEXT: s_mov_b64 s[14:15], s[38:39]
-; CHECK-NEXT: s_mov_b64 s[16:17], s[40:41]
-; CHECK-NEXT: s_mov_b64 s[18:19], s[42:43]
-; CHECK-NEXT: ; kill: killed $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
-; CHECK-NEXT: ; kill: killed $sgpr8_sgpr9_sgpr10 killed $sgpr11
-; CHECK-NEXT: ; kill: killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_sub_f32_e32 v1, v4, v3
; CHECK-NEXT: v_mul_f32_e32 v0, v1, v0
@@ -331,47 +177,52 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_mov_b64 vcc, vcc
; CHECK-NEXT: s_cbranch_vccz .LBB0_8
; CHECK-NEXT: .LBB0_9: ; %Flow13
-; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_or_b64 exec, exec, s[14:15]
; CHECK-NEXT: .LBB0_10: ; %UnifiedReturnBlock
-; CHECK-NEXT: s_or_b64 exec, exec, s[20:21]
-; CHECK-NEXT: v_readlane_b32 s67, v5, 33
-; CHECK-NEXT: v_readlane_b32 s66, v5, 32
-; CHECK-NEXT: v_readlane_b32 s65, v5, 31
-; CHECK-NEXT: v_readlane_b32 s64, v5, 30
-; CHECK-NEXT: v_readlane_b32 s63, v5, 29
-; CHECK-NEXT: v_readlane_b32 s62, v5, 28
-; CHECK-NEXT: v_readlane_b32 s61, v5, 27
-; CHECK-NEXT: v_readlane_b32 s60, v5, 26
-; CHECK-NEXT: v_readlane_b32 s59, v5, 25
-; CHECK-NEXT: v_readlane_b32 s58, v5, 24
-; CHECK-NEXT: v_readlane_b32 s57, v5, 23
-; CHECK-NEXT: v_readlane_b32 s56, v5, 22
-; CHECK-NEXT: v_readlane_b32 s55, v5, 21
-; CHECK-NEXT: v_readlane_b32 s54, v5, 20
-; CHECK-NEXT: v_readlane_b32 s53, v5, 19
-; CHECK-NEXT: v_readlane_b32 s52, v5, 18
-; CHECK-NEXT: v_readlane_b32 s51, v5, 17
-; CHECK-NEXT: v_readlane_b32 s50, v5, 16
-; CHECK-NEXT: v_readlane_b32 s49, v5, 15
-; CHECK-NEXT: v_readlane_b32 s48, v5, 14
-; CHECK-NEXT: v_readlane_b32 s47, v5, 13
-; CHECK-NEXT: v_readlane_b32 s46, v5, 12
-; CHECK-NEXT: v_readlane_b32 s45, v5, 11
-; CHECK-NEXT: v_readlane_b32 s44, v5, 10
-; CHECK-NEXT: v_readlane_b32 s43, v5, 9
-; CHECK-NEXT: v_readlane_b32 s42, v5, 8
-; CHECK-NEXT: v_readlane_b32 s41, v5, 7
-; CHECK-NEXT: v_readlane_b32 s40, v5, 6
-; CHECK-NEXT: v_readlane_b32 s39, v5, 5
-; CHECK-NEXT: v_readlane_b32 s38, v5, 4
-; CHECK-NEXT: v_readlane_b32 s37, v5, 3
-; CHECK-NEXT: v_readlane_b32 s36, v5, 2
-; CHECK-NEXT: v_readlane_b32 s31, v5, 1
-; CHECK-NEXT: v_readlane_b32 s30, v5, 0
+; CHECK-NEXT: s_or_b64 exec, exec, s[12:13]
+; CHECK-NEXT: v_readlane_b32 s71, v6, 39
+; CHECK-NEXT: v_readlane_b32 s70, v6, 38
+; CHECK-NEXT: v_readlane_b32 s69, v6, 37
+; CHECK-NEXT: v_readlane_b32 s68, v6, 36
+; CHECK-NEXT: v_readlane_b32 s67, v6, 35
+; CHECK-NEXT: v_readlane_b32 s66, v6, 34
+; CHECK-NEXT: v_readlane_b32 s65, v6, 33
+; CHECK-NEXT: v_readlane_b32 s64, v6, 32
+; CHECK-NEXT: v_readlane_b32 s63, v6, 31
+; CHECK-NEXT: v_readlane_b32 s62, v6, 30
+; CHECK-NEXT: v_readlane_b32 s61, v6, 29
+; CHECK-NEXT: v_readlane_b32 s60, v6, 28
+; CHECK-NEXT: v_readlane_b32 s59, v6, 27
+; CHECK-NEXT: v_readlane_b32 s58, v6, 26
+; CHECK-NEXT: v_readlane_b32 s57, v6, 25
+; CHECK-NEXT: v_readlane_b32 s56, v6, 24
+; CHECK-NEXT: v_readlane_b32 s55, v6, 23
+; CHECK-NEXT: v_readlane_b32 s54, v6, 22
+; CHECK-NEXT: v_readlane_b32 s53, v6, 21
+; CHECK-NEXT: v_readlane_b32 s52, v6, 20
+; CHECK-NEXT: v_readlane_b32 s51, v6, 19
+; CHECK-NEXT: v_readlane_b32 s50, v6, 18
+; CHECK-NEXT: v_readlane_b32 s49, v6, 17
+; CHECK-NEXT: v_readlane_b32 s48, v6, 16
+; CHECK-NEXT: v_readlane_b32 s47, v6, 15
+; CHECK-NEXT: v_readlane_b32 s46, v6, 14
+; CHECK-NEXT: v_readlane_b32 s45, v6, 13
+; CHECK-NEXT: v_readlane_b32 s44, v6, 12
+; CHECK-NEXT: v_readlane_b32 s43, v6, 11
+; CHECK-NEXT: v_readlane_b32 s42, v6, 10
+; CHECK-NEXT: v_readlane_b32 s41, v6, 9
+; CHECK-NEXT: v_readlane_b32 s40, v6, 8
+; CHECK-NEXT: v_readlane_b32 s39, v6, 7
+; CHECK-NEXT: v_readlane_b32 s38, v6, 6
+; CHECK-NEXT: v_readlane_b32 s37, v6, 5
+; CHECK-NEXT: v_readlane_b32 s36, v6, 4
+; CHECK-NEXT: v_readlane_b32 s35, v6, 3
+; CHECK-NEXT: v_readlane_b32 s34, v6, 2
+; CHECK-NEXT: v_readlane_b32 s31, v6, 1
+; CHECK-NEXT: v_readlane_b32 s30, v6, 0
; CHECK-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/imm.ll b/llvm/test/CodeGen/AMDGPU/imm.ll
index a328bbe8b4ddc..42168354897e5 100644
--- a/llvm/test/CodeGen/AMDGPU/imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/imm.ll
@@ -1532,8 +1532,8 @@ define amdgpu_kernel void @store_inline_imm_0.0_f64(ptr addrspace(1) %out) {
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_mov_b32_e32 v0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v1, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1543,8 +1543,8 @@ define amdgpu_kernel void @store_inline_imm_0.0_f64(ptr addrspace(1) %out) {
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 0
; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 74020c43a3ca3..773988227be65 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -9547,78 +9547,77 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: s_mov_b32 s19, 0xe8f000
; NOOPT-NEXT: s_add_u32 s16, s16, s5
; NOOPT-NEXT: s_addc_u32 s17, s17, 0
-; NOOPT-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; NOOPT-NEXT: v_writelane_b32 v33, s4, 0
-; NOOPT-NEXT: s_mov_b32 s4, s1
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 0
-; NOOPT-NEXT: v_writelane_b32 v33, s4, 1
-; NOOPT-NEXT: s_mov_b32 s4, s0
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 1
+; NOOPT-NEXT: s_mov_b32 s5, s4
+; NOOPT-NEXT: s_mov_b32 s4, s3
+; NOOPT-NEXT: s_mov_b32 s6, s2
+; NOOPT-NEXT: s_mov_b32 s7, s1
; NOOPT-NEXT: buffer_store_dword v4, off, s[16:19], 0 offset:144 ; 4-byte Folded Spill
; NOOPT-NEXT: v_mov_b32_e32 v2, v1
-; NOOPT-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
-; NOOPT-NEXT: s_mov_b32 s5, s0
-; NOOPT-NEXT: s_mov_b32 s6, s2
-; NOOPT-NEXT: s_mov_b32 s7, s3
-; NOOPT-NEXT: ; implicit-def: $sgpr0
-; NOOPT-NEXT: ; implicit-def: $sgpr0
+; NOOPT-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; NOOPT-NEXT: s_mov_b32 s1, s7
+; NOOPT-NEXT: s_mov_b32 s2, s6
+; NOOPT-NEXT: s_mov_b32 s3, s4
+; NOOPT-NEXT: ; implicit-def: $sgpr4
+; NOOPT-NEXT: ; implicit-def: $sgpr4
; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v1, v2
; NOOPT-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:136 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[16:19], 0 offset:140 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b32 s8, 0xf000
-; NOOPT-NEXT: s_mov_b32 s0, 0
-; NOOPT-NEXT: v_writelane_b32 v33, s0, 2
-; NOOPT-NEXT: s_mov_b32 s2, s0
-; NOOPT-NEXT: s_mov_b32 s3, s8
-; NOOPT-NEXT: s_mov_b32 s8, s0
-; NOOPT-NEXT: s_mov_b32 s9, s0
+; NOOPT-NEXT: s_mov_b32 s4, 0
+; NOOPT-NEXT: s_mov_b32 s6, s4
+; NOOPT-NEXT: s_mov_b32 s7, s8
+; NOOPT-NEXT: s_mov_b32 s8, s4
+; NOOPT-NEXT: s_mov_b32 s9, s4
; NOOPT-NEXT: ; kill: def $sgpr8_sgpr9 killed $sgpr8_sgpr9 def $sgpr8_sgpr9_sgpr10_sgpr11
-; NOOPT-NEXT: s_mov_b64 s[10:11], s[2:3]
-; NOOPT-NEXT: v_writelane_b32 v33, s8, 3
-; NOOPT-NEXT: v_writelane_b32 v33, s9, 4
-; NOOPT-NEXT: v_writelane_b32 v33, s10, 5
-; NOOPT-NEXT: v_writelane_b32 v33, s11, 6
-; NOOPT-NEXT: ; kill: def $sgpr8_sgpr9_sgpr10_sgpr11 killed $sgpr4_sgpr5_sgpr6_sgpr7
-; NOOPT-NEXT: ; implicit-def: $sgpr2_sgpr3
+; NOOPT-NEXT: s_mov_b64 s[10:11], s[6:7]
+; NOOPT-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
+; NOOPT-NEXT: v_writelane_b32 v33, s8, 0
+; NOOPT-NEXT: v_writelane_b32 v33, s9, 1
+; NOOPT-NEXT: v_writelane_b32 v33, s10, 2
+; NOOPT-NEXT: v_writelane_b32 v33, s11, 3
+; NOOPT-NEXT: ; kill: def $sgpr8_sgpr9_sgpr10_sgpr11 killed $sgpr0_sgpr1_sgpr2_sgpr3
+; NOOPT-NEXT: ; implicit-def: $sgpr6_sgpr7
; NOOPT-NEXT: s_waitcnt expcnt(1)
-; NOOPT-NEXT: v_mov_b32_e32 v0, s1
-; NOOPT-NEXT: buffer_load_dword v0, v0, s[4:7], s0 offen
+; NOOPT-NEXT: v_mov_b32_e32 v0, s5
+; NOOPT-NEXT: buffer_load_dword v0, v0, s[0:3], s4 offen
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:132 ; 4-byte Folded Spill
+; NOOPT-NEXT: s_mov_b64 s[0:1], 0
+; NOOPT-NEXT: v_writelane_b32 v33, s0, 4
+; NOOPT-NEXT: v_writelane_b32 v33, s1, 5
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v0, s0
-; NOOPT-NEXT: v_mov_b32_e32 v30, s0
-; NOOPT-NEXT: v_mov_b32_e32 v29, s0
+; NOOPT-NEXT: v_mov_b32_e32 v1, s1
+; NOOPT-NEXT: v_mov_b32_e32 v29, s1
; NOOPT-NEXT: v_mov_b32_e32 v28, s0
-; NOOPT-NEXT: v_mov_b32_e32 v27, s0
+; NOOPT-NEXT: v_mov_b32_e32 v27, s1
; NOOPT-NEXT: v_mov_b32_e32 v26, s0
-; NOOPT-NEXT: v_mov_b32_e32 v25, s0
+; NOOPT-NEXT: v_mov_b32_e32 v25, s1
; NOOPT-NEXT: v_mov_b32_e32 v24, s0
-; NOOPT-NEXT: v_mov_b32_e32 v23, s0
+; NOOPT-NEXT: v_mov_b32_e32 v23, s1
; NOOPT-NEXT: v_mov_b32_e32 v22, s0
-; NOOPT-NEXT: v_mov_b32_e32 v21, s0
+; NOOPT-NEXT: v_mov_b32_e32 v21, s1
; NOOPT-NEXT: v_mov_b32_e32 v20, s0
-; NOOPT-NEXT: v_mov_b32_e32 v19, s0
+; NOOPT-NEXT: v_mov_b32_e32 v19, s1
; NOOPT-NEXT: v_mov_b32_e32 v18, s0
-; NOOPT-NEXT: v_mov_b32_e32 v17, s0
+; NOOPT-NEXT: v_mov_b32_e32 v17, s1
; NOOPT-NEXT: v_mov_b32_e32 v16, s0
-; NOOPT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
-; NOOPT-NEXT: v_mov_b32_e32 v1, v30
-; NOOPT-NEXT: v_mov_b32_e32 v2, v29
-; NOOPT-NEXT: v_mov_b32_e32 v3, v28
-; NOOPT-NEXT: v_mov_b32_e32 v4, v27
-; NOOPT-NEXT: v_mov_b32_e32 v5, v26
-; NOOPT-NEXT: v_mov_b32_e32 v6, v25
-; NOOPT-NEXT: v_mov_b32_e32 v7, v24
-; NOOPT-NEXT: v_mov_b32_e32 v8, v23
-; NOOPT-NEXT: v_mov_b32_e32 v9, v22
-; NOOPT-NEXT: v_mov_b32_e32 v10, v21
-; NOOPT-NEXT: v_mov_b32_e32 v11, v20
-; NOOPT-NEXT: v_mov_b32_e32 v12, v19
-; NOOPT-NEXT: v_mov_b32_e32 v13, v18
-; NOOPT-NEXT: v_mov_b32_e32 v14, v17
-; NOOPT-NEXT: v_mov_b32_e32 v15, v16
+; NOOPT-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; NOOPT-NEXT: v_mov_b32_e32 v2, v28
+; NOOPT-NEXT: v_mov_b32_e32 v3, v29
+; NOOPT-NEXT: v_mov_b32_e32 v4, v26
+; NOOPT-NEXT: v_mov_b32_e32 v5, v27
+; NOOPT-NEXT: v_mov_b32_e32 v6, v24
+; NOOPT-NEXT: v_mov_b32_e32 v7, v25
+; NOOPT-NEXT: v_mov_b32_e32 v8, v22
+; NOOPT-NEXT: v_mov_b32_e32 v9, v23
+; NOOPT-NEXT: v_mov_b32_e32 v10, v20
+; NOOPT-NEXT: v_mov_b32_e32 v11, v21
+; NOOPT-NEXT: v_mov_b32_e32 v12, v18
+; NOOPT-NEXT: v_mov_b32_e32 v13, v19
+; NOOPT-NEXT: v_mov_b32_e32 v14, v16
+; NOOPT-NEXT: v_mov_b32_e32 v15, v17
; NOOPT-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:68 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[16:19], 0 offset:72 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v2, off, s[16:19], 0 offset:76 ; 4-byte Folded Spill
@@ -9636,8 +9635,8 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_store_dword v14, off, s[16:19], 0 offset:124 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v15, off, s[16:19], 0 offset:128 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[0:1], exec
-; NOOPT-NEXT: v_writelane_b32 v33, s0, 7
-; NOOPT-NEXT: v_writelane_b32 v33, s1, 8
+; NOOPT-NEXT: v_writelane_b32 v33, s0, 6
+; NOOPT-NEXT: v_writelane_b32 v33, s1, 7
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
; NOOPT-NEXT: buffer_store_dword v33, off, s[16:19], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
@@ -9664,8 +9663,8 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 9
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 10
+; NOOPT-NEXT: v_readlane_b32 s0, v33, 8
+; NOOPT-NEXT: v_readlane_b32 s1, v33, 9
; NOOPT-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
@@ -9730,8 +9729,8 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_store_dword v14, off, s[16:19], 0 offset:60 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v15, off, s[16:19], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
-; NOOPT-NEXT: v_writelane_b32 v33, s2, 9
-; NOOPT-NEXT: v_writelane_b32 v33, s3, 10
+; NOOPT-NEXT: v_writelane_b32 v33, s2, 8
+; NOOPT-NEXT: v_writelane_b32 v33, s3, 9
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
; NOOPT-NEXT: buffer_store_dword v33, off, s[16:19], 0 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
@@ -9743,18 +9742,18 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 7
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 8
+; NOOPT-NEXT: v_readlane_b32 s0, v33, 6
+; NOOPT-NEXT: v_readlane_b32 s1, v33, 7
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 3
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 4
-; NOOPT-NEXT: v_readlane_b32 s2, v33, 5
-; NOOPT-NEXT: v_readlane_b32 s3, v33, 6
+; NOOPT-NEXT: v_readlane_b32 s0, v33, 0
+; NOOPT-NEXT: v_readlane_b32 s1, v33, 1
+; NOOPT-NEXT: v_readlane_b32 s2, v33, 2
+; NOOPT-NEXT: v_readlane_b32 s3, v33, 3
; NOOPT-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v5, off, s[16:19], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:148 ; 4-byte Folded Reload
@@ -9842,26 +9841,26 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; SI-MOVREL: ; %bb.0: ; %entry
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, s4
; SI-MOVREL-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; SI-MOVREL-NEXT: s_mov_b32 s2, 0
; SI-MOVREL-NEXT: v_mov_b32_e32 v5, 0
+; SI-MOVREL-NEXT: v_mov_b32_e32 v6, 0
+; SI-MOVREL-NEXT: s_mov_b32 s2, 0
+; SI-MOVREL-NEXT: v_mov_b32_e32 v8, v6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v10, v6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v12, v6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v14, v6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v16, v6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v18, v6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v20, v6
; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: s_mov_b32 s0, s2
; SI-MOVREL-NEXT: s_mov_b32 s1, s2
-; SI-MOVREL-NEXT: v_mov_b32_e32 v6, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v7, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v8, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v9, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v10, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v11, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v12, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v13, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v14, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v15, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v16, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v17, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v18, v5
; SI-MOVREL-NEXT: v_mov_b32_e32 v19, v5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v20, v5
; SI-MOVREL-NEXT: s_mov_b64 s[4:5], exec
; SI-MOVREL-NEXT: .LBB27_1: ; =>This Inner Loop Header: Depth=1
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
@@ -9885,21 +9884,21 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, s4
; VI-MOVREL-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v6, v5
+; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v8, v6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v10, v6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v12, v6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v14, v6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, v6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v18, v6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v20, v6
; VI-MOVREL-NEXT: v_mov_b32_e32 v7, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v8, v5
; VI-MOVREL-NEXT: v_mov_b32_e32 v9, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v10, v5
; VI-MOVREL-NEXT: v_mov_b32_e32 v11, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v12, v5
; VI-MOVREL-NEXT: v_mov_b32_e32 v13, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v14, v5
; VI-MOVREL-NEXT: v_mov_b32_e32 v15, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v16, v5
; VI-MOVREL-NEXT: v_mov_b32_e32 v17, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v18, v5
; VI-MOVREL-NEXT: v_mov_b32_e32 v19, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v20, v5
; VI-MOVREL-NEXT: s_mov_b64 s[0:1], exec
; VI-MOVREL-NEXT: .LBB27_1: ; =>This Inner Loop Header: Depth=1
; VI-MOVREL-NEXT: s_waitcnt vmcnt(0)
@@ -9929,21 +9928,21 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, s4
; VI-IDXMODE-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, v5
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, v6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v10, v6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, v6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, v6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, v6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v18, v6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v20, v6
; VI-IDXMODE-NEXT: v_mov_b32_e32 v7, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, v5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v10, v5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v11, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, v5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, v5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, v5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v18, v5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v19, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v20, v5
; VI-IDXMODE-NEXT: s_mov_b64 s[0:1], exec
; VI-IDXMODE-NEXT: .LBB27_1: ; =>This Inner Loop Header: Depth=1
; VI-IDXMODE-NEXT: s_waitcnt vmcnt(0)
@@ -9974,21 +9973,21 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, s4
; GFX9-IDXMODE-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v6, v5
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v8, v6
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v10, v6
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v12, v6
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v14, v6
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, v6
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v18, v6
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v20, v6
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v8, v5
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v10, v5
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v11, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v12, v5
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v13, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v14, v5
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v15, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, v5
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v17, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v18, v5
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v19, v5
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v20, v5
; GFX9-IDXMODE-NEXT: s_mov_b64 s[0:1], exec
; GFX9-IDXMODE-NEXT: .LBB27_1: ; =>This Inner Loop Header: Depth=1
; GFX9-IDXMODE-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index f961e857f39e5..37990085e6abf 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -15,11 +15,9 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:16
; SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v8, 0
-; SDAG-NEXT: s_mov_b32 s12, 0
+; SDAG-NEXT: s_mov_b64 s[12:13], 0
; SDAG-NEXT: s_mov_b32 s3, exec_lo
-; SDAG-NEXT: s_mov_b32 s13, s12
-; SDAG-NEXT: s_mov_b32 s14, s12
-; SDAG-NEXT: s_mov_b32 s15, s12
+; SDAG-NEXT: s_mov_b64 s[14:15], s[12:13]
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_readfirstlane_b32 s4, v0
@@ -51,14 +49,10 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: s_mov_b32 exec_lo, s3
; SDAG-NEXT: v_dual_mov_b32 v0, 0x7fc00000 :: v_dual_mov_b32 v1, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
-; SDAG-NEXT: s_mov_b32 s0, s12
-; SDAG-NEXT: s_mov_b32 s1, s12
-; SDAG-NEXT: s_mov_b32 s2, s12
-; SDAG-NEXT: s_mov_b32 s3, s12
-; SDAG-NEXT: s_mov_b32 s4, s12
-; SDAG-NEXT: s_mov_b32 s5, s12
-; SDAG-NEXT: s_mov_b32 s6, s12
-; SDAG-NEXT: s_mov_b32 s7, s12
+; SDAG-NEXT: s_mov_b64 s[0:1], s[12:13]
+; SDAG-NEXT: s_mov_b64 s[2:3], s[12:13]
+; SDAG-NEXT: s_mov_b64 s[4:5], s[12:13]
+; SDAG-NEXT: s_mov_b64 s[6:7], s[12:13]
; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: image_sample_c_lz v0, [v1, v1, v0, v1], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: image_sample_c_lz v3, [v1, v1, v1, v1], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll b/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll
index 7caa563d8b298..6a5a7cc3e6219 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-need-live-out-undef-subregister-def.ll
@@ -8,12 +8,10 @@ define amdgpu_vs void @test(i32 inreg %cmp, i32 %e0) {
; CHECK-LABEL: test:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_cmp_eq_u32 s0, 0
-; CHECK-NEXT: s_mov_b32 s0, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_2
; CHECK-NEXT: ; %bb.1: ; %load
-; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_mov_b32 s2, s0
-; CHECK-NEXT: s_mov_b32 s3, s0
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_mov_b64 s[2:3], s[0:1]
; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: buffer_load_format_xy v[1:2], v1, s[0:3], 0 idxen
; CHECK-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 8732c77778b01..65bc36644fdad 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -464,7 +464,9 @@ bb:
; GCN-LABEL: {{^}}test_mfma_f32_32x32x1f32_imm_splat:
; GCN-DAG: v_mov_b32_e32 [[TWO:v[0-9]+]], 2.0
; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1.0
-; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0
+; NOLIT-SRCC-DAG: v_mov_b32_e32 v0, 0
+; NOLIT-SRCC-DAG: v_mov_b32_e32 v1, 0
+; NOLIT-SRCC-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
; NOLIT-SRCC: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], a[{{[0-9:]+}}]
; LIT-SRCC: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 0
; GFX90A: v_mfma_f32_32x32x1f32 a[{{[0-9]+:[0-9]+}}], [[ONE]], [[TWO]], 0
diff --git a/llvm/test/CodeGen/AMDGPU/max-hard-clause-length.ll b/llvm/test/CodeGen/AMDGPU/max-hard-clause-length.ll
index af713179a888d..4d5b532d3c90b 100644
--- a/llvm/test/CodeGen/AMDGPU/max-hard-clause-length.ll
+++ b/llvm/test/CodeGen/AMDGPU/max-hard-clause-length.ll
@@ -11,11 +11,9 @@ define amdgpu_kernel void @long_store_chain(ptr addrspace(1) %p) {
; GFX10-LABEL: long_store_chain:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s0, 0
+; GFX10-NEXT: s_mov_b64 s[0:1], 0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_mov_b32 s2, s0
-; GFX10-NEXT: s_mov_b32 s3, s0
+; GFX10-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s2
@@ -92,11 +90,9 @@ define amdgpu_kernel void @long_store_chain(ptr addrspace(1) %p) {
; GFX11-LABEL: long_store_chain:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_mov_b32 s2, s0
-; GFX11-NEXT: s_mov_b32 s3, s0
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -175,11 +171,9 @@ define amdgpu_kernel void @long_store_chain(ptr addrspace(1) %p) {
; GFX12-LABEL: long_store_chain:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index 8157b1a7f7c80..8a8c423ba113a 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -175,9 +175,9 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: .LBB1_8: ; %post-loop-memcpy-expansion
; CHECK-NEXT: v_mov_b32_e32 v2, 0
-; CHECK-NEXT: v_mov_b32_e32 v3, v2
+; CHECK-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v5, v2
; CHECK-NEXT: s_and_b64 vcc, exec, 0
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: .LBB1_9: ; %loop-memcpy-expansion2
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
index 9d329a2d121ed..b611d63213253 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
@@ -457,7 +457,9 @@ exit:
; GCN-LABEL: {{^}}test_mfma_loop_mfma_forward_init:
-; GFX908-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, 0
+; GFX908: v_mov_b32_e32 v0, 0
+; GFX908: v_mov_b32_e32 v1, 0
+; GFX908-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
; GFX908: v_mfma_f32_32x32x1f32 a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, a[{{[0-9:]+}}]
; GFX90A-NOT: v_accvgpr
; GFX90A: v_mfma_f32_32x32x1f32 a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, 0{{$}}
@@ -498,7 +500,9 @@ exit:
; GCN-LABEL: {{^}}test_mfma_loop_agpr_init:
-; GFX908-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, 0
+; GFX908: v_mov_b32_e32 v0, 0
+; GFX908: v_mov_b32_e32 v1, 0
+; GFX908-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}}
; GFX908: v_mfma_f32_32x32x1f32 a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, a[{{[0-9:]+}}]
; GFX90A-NOT: v_accvgpr
; GFX90A: v_mfma_f32_32x32x1f32 a[{{[0-9:]+}}], v{{[0-9]+}}, v{{[0-9]+}}, 0{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll b/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll
index 43e3a1fa29483..890c7234b2e45 100644
--- a/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll
+++ b/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll
@@ -28,20 +28,22 @@ define void @nonkernel() {
; GFX9-LABEL: nonkernel:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: ds_write_b32 v0, v0 offset:8
-; GFX9-NEXT: ds_write_b64 v0, v[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: ds_write_b32 v2, v2 offset:8
+; GFX9-NEXT: ds_write_b64 v2, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: nonkernel:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: ds_write_b32 v0, v0 offset:8
-; GFX10-NEXT: ds_write_b64 v0, v[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: ds_write_b32 v2, v2 offset:8
+; GFX10-NEXT: ds_write_b64 v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/no-dup-inst-prefetch.ll b/llvm/test/CodeGen/AMDGPU/no-dup-inst-prefetch.ll
index d62f045674ace..4b58fdab44bc6 100644
--- a/llvm/test/CodeGen/AMDGPU/no-dup-inst-prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-dup-inst-prefetch.ll
@@ -8,7 +8,7 @@ define amdgpu_cs void @_amdgpu_cs_main(float %0, i32 %1) {
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b64 s[4:5], 0
; GFX10-NEXT: s_mov_b32 s1, 0
; GFX10-NEXT: ; implicit-def: $sgpr2
; GFX10-NEXT: s_inst_prefetch 0x1
@@ -29,17 +29,11 @@ define amdgpu_cs void @_amdgpu_cs_main(float %0, i32 %1) {
; GFX10-NEXT: s_cbranch_execz .LBB0_1
; GFX10-NEXT: ; %bb.3: ; %branch2_merge
; GFX10-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; GFX10-NEXT: s_mov_b32 s5, s4
-; GFX10-NEXT: s_mov_b32 s6, s4
-; GFX10-NEXT: s_mov_b32 s7, s4
-; GFX10-NEXT: s_mov_b32 s8, s4
-; GFX10-NEXT: s_mov_b32 s9, s4
-; GFX10-NEXT: s_mov_b32 s10, s4
-; GFX10-NEXT: s_mov_b32 s11, s4
-; GFX10-NEXT: s_mov_b32 s12, s4
-; GFX10-NEXT: s_mov_b32 s13, s4
-; GFX10-NEXT: s_mov_b32 s14, s4
-; GFX10-NEXT: s_mov_b32 s15, s4
+; GFX10-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX10-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX10-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX10-NEXT: s_mov_b64 s[12:13], s[4:5]
+; GFX10-NEXT: s_mov_b64 s[14:15], s[4:5]
; GFX10-NEXT: s_andn2_b32 s2, s2, exec_lo
; GFX10-NEXT: image_sample_lz v1, [v2, v2, v1], s[8:15], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_3D
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -57,7 +51,7 @@ define amdgpu_cs void @_amdgpu_cs_main(float %0, i32 %1) {
; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_mov_b32 s4, 0
+; GFX12-NEXT: s_mov_b64 s[4:5], 0
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: ; implicit-def: $sgpr2
; GFX12-NEXT: s_branch .LBB0_2
@@ -77,17 +71,11 @@ define amdgpu_cs void @_amdgpu_cs_main(float %0, i32 %1) {
; GFX12-NEXT: s_cbranch_execz .LBB0_1
; GFX12-NEXT: ; %bb.3: ; %branch2_merge
; GFX12-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: s_mov_b32 s6, s4
-; GFX12-NEXT: s_mov_b32 s7, s4
-; GFX12-NEXT: s_mov_b32 s8, s4
-; GFX12-NEXT: s_mov_b32 s9, s4
-; GFX12-NEXT: s_mov_b32 s10, s4
-; GFX12-NEXT: s_mov_b32 s11, s4
-; GFX12-NEXT: s_mov_b32 s12, s4
-; GFX12-NEXT: s_mov_b32 s13, s4
-; GFX12-NEXT: s_mov_b32 s14, s4
-; GFX12-NEXT: s_mov_b32 s15, s4
+; GFX12-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX12-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX12-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX12-NEXT: s_mov_b64 s[12:13], s[4:5]
+; GFX12-NEXT: s_mov_b64 s[14:15], s[4:5]
; GFX12-NEXT: s_and_not1_b32 s2, s2, exec_lo
; GFX12-NEXT: image_sample_lz v1, [v2, v2, v1], s[8:15], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_3D
; GFX12-NEXT: s_wait_samplecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index a9b8663a48dea..d0d199fb02835 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -7,9 +7,9 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v1, 0
-; GFX942-NEXT: s_mov_b32 s2, 0
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_accvgpr_write_b32 a0, v1
-; GFX942-NEXT: s_mov_b32 s3, 0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_lg_u32 s0, 0
; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
@@ -18,10 +18,7 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX942-NEXT: s_branch .LBB0_2
; GFX942-NEXT: .LBB0_1: ; %bb2
; GFX942-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; GFX942-NEXT: s_or_b32 s4, s3, 1
-; GFX942-NEXT: s_ashr_i32 s5, s3, 31
-; GFX942-NEXT: s_mov_b32 s3, s2
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX942-NEXT: s_nop 2
; GFX942-NEXT: v_accvgpr_read_b32 v0, a0
; GFX942-NEXT: v_mov_b32_e32 v2, v1
; GFX942-NEXT: v_mov_b32_e32 v3, v1
@@ -29,16 +26,17 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX942-NEXT: v_accvgpr_write_b32 a1, v1
; GFX942-NEXT: v_accvgpr_write_b32 a2, v2
; GFX942-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX942-NEXT: s_and_b32 s3, s5, s4
-; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: s_or_b32 s2, s4, 1
+; GFX942-NEXT: s_ashr_i32 s3, s4, 31
; GFX942-NEXT: v_mfma_f32_16x16x16_f16 a[0:3], v[4:5], v[4:5], a[0:3]
+; GFX942-NEXT: s_and_b32 s4, s3, s2
; GFX942-NEXT: s_cbranch_execz .LBB0_4
; GFX942-NEXT: .LBB0_2: ; %bb
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX942-NEXT: s_cbranch_vccz .LBB0_1
; GFX942-NEXT: ; %bb.3:
-; GFX942-NEXT: ; implicit-def: $sgpr3
+; GFX942-NEXT: ; implicit-def: $sgpr4
; GFX942-NEXT: .LBB0_4: ; %common.ret
; GFX942-NEXT: s_endpgm
;
@@ -46,39 +44,37 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX908: ; %bb.0: ; %entry
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX908-NEXT: v_mov_b32_e32 v1, 0
-; GFX908-NEXT: s_mov_b32 s2, 0
-; GFX908-NEXT: s_mov_b32 s3, 0
+; GFX908-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-NEXT: s_mov_b32 s4, 0
; GFX908-NEXT: v_accvgpr_write_b32 a0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_cmp_lg_u32 s0, 0
; GFX908-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX908-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX908-NEXT: v_mov_b32_e32 v5, 0
; GFX908-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
; GFX908-NEXT: s_branch .LBB0_2
; GFX908-NEXT: .LBB0_1: ; %bb2
; GFX908-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; GFX908-NEXT: s_or_b32 s4, s3, 1
-; GFX908-NEXT: s_ashr_i32 s5, s3, 31
-; GFX908-NEXT: s_mov_b32 s3, s2
-; GFX908-NEXT: s_nop 3
+; GFX908-NEXT: s_nop 4
; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-NEXT: v_mov_b32_e32 v5, s3
-; GFX908-NEXT: v_mov_b32_e32 v4, s2
; GFX908-NEXT: v_mov_b32_e32 v2, v1
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX908-NEXT: s_and_b32 s3, s5, s4
+; GFX908-NEXT: s_or_b32 s2, s4, 1
; GFX908-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[4:5], v[4:5], a[0:3]
+; GFX908-NEXT: s_ashr_i32 s3, s4, 31
+; GFX908-NEXT: s_and_b32 s4, s3, s2
; GFX908-NEXT: s_cbranch_execz .LBB0_4
; GFX908-NEXT: .LBB0_2: ; %bb
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX908-NEXT: s_cbranch_vccz .LBB0_1
; GFX908-NEXT: ; %bb.3:
-; GFX908-NEXT: ; implicit-def: $sgpr3
+; GFX908-NEXT: ; implicit-def: $sgpr4
; GFX908-NEXT: .LBB0_4: ; %common.ret
; GFX908-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
index 81d792183dc06..b2f2816e7f82b 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -global-isel -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1150 -global-isel -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GIGFX1150 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,GIGFX12 %s
define amdgpu_vs float @fadd_f32(float inreg %a, float inreg %b) {
; CHECK-LABEL: fadd_f32:
@@ -45,12 +45,26 @@ define amdgpu_vs float @fmin_f32(float inreg %a, float inreg %b) {
; GFX1150-NEXT: v_mov_b32_e32 v0, s0
; GFX1150-NEXT: ; return to shader part epilog
;
+; GIGFX1150-LABEL: fmin_f32:
+; GIGFX1150: ; %bb.0:
+; GIGFX1150-NEXT: s_min_f32 s0, s0, s1
+; GIGFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX1150-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX1150-NEXT: ; return to shader part epilog
+;
; GFX12-LABEL: fmin_f32:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_min_num_f32 s0, s0, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
+;
+; GIGFX12-LABEL: fmin_f32:
+; GIGFX12: ; %bb.0:
+; GIGFX12-NEXT: s_min_num_f32 s0, s0, s1
+; GIGFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX12-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX12-NEXT: ; return to shader part epilog
%min = call float @llvm.minnum.f32(float %a, float %b)
ret float %min
}
@@ -63,12 +77,26 @@ define amdgpu_vs float @fmax_f32(float inreg %a, float inreg %b) {
; GFX1150-NEXT: v_mov_b32_e32 v0, s0
; GFX1150-NEXT: ; return to shader part epilog
;
+; GIGFX1150-LABEL: fmax_f32:
+; GIGFX1150: ; %bb.0:
+; GIGFX1150-NEXT: s_max_f32 s0, s0, s1
+; GIGFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX1150-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX1150-NEXT: ; return to shader part epilog
+;
; GFX12-LABEL: fmax_f32:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_max_num_f32 s0, s0, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
+;
+; GIGFX12-LABEL: fmax_f32:
+; GIGFX12: ; %bb.0:
+; GIGFX12-NEXT: s_max_num_f32 s0, s0, s1
+; GIGFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX12-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX12-NEXT: ; return to shader part epilog
%max = call float @llvm.maxnum.f32(float %a, float %b)
ret float %max
}
@@ -114,12 +142,26 @@ define amdgpu_vs half @fmin_f16(half inreg %a, half inreg %b) {
; GFX1150-NEXT: v_mov_b32_e32 v0, s0
; GFX1150-NEXT: ; return to shader part epilog
;
+; GIGFX1150-LABEL: fmin_f16:
+; GIGFX1150: ; %bb.0:
+; GIGFX1150-NEXT: s_min_f16 s0, s0, s1
+; GIGFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX1150-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX1150-NEXT: ; return to shader part epilog
+;
; GFX12-LABEL: fmin_f16:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_min_num_f16 s0, s0, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
+;
+; GIGFX12-LABEL: fmin_f16:
+; GIGFX12: ; %bb.0:
+; GIGFX12-NEXT: s_min_num_f16 s0, s0, s1
+; GIGFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX12-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX12-NEXT: ; return to shader part epilog
%min = call half @llvm.minnum.f16(half %a, half %b)
ret half %min
}
@@ -132,12 +174,26 @@ define amdgpu_vs half @fmax_f16(half inreg %a, half inreg %b) {
; GFX1150-NEXT: v_mov_b32_e32 v0, s0
; GFX1150-NEXT: ; return to shader part epilog
;
+; GIGFX1150-LABEL: fmax_f16:
+; GIGFX1150: ; %bb.0:
+; GIGFX1150-NEXT: s_max_f16 s0, s0, s1
+; GIGFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX1150-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX1150-NEXT: ; return to shader part epilog
+;
; GFX12-LABEL: fmax_f16:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_max_num_f16 s0, s0, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
+;
+; GIGFX12-LABEL: fmax_f16:
+; GIGFX12: ; %bb.0:
+; GIGFX12-NEXT: s_max_num_f16 s0, s0, s1
+; GIGFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX12-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX12-NEXT: ; return to shader part epilog
%max = call half @llvm.maxnum.f16(half %a, half %b)
ret half %max
}
@@ -211,31 +267,53 @@ define amdgpu_vs half @fmac_f16_with_mov(half inreg %a, half inreg %b, half inre
define amdgpu_ps float @_amdgpu_ps_main() {
; GFX1150-LABEL: _amdgpu_ps_main:
; GFX1150: ; %bb.0: ; %bb
-; GFX1150-NEXT: s_mov_b32 s0, 0
-; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1150-NEXT: s_mov_b32 s1, s0
-; GFX1150-NEXT: s_mov_b32 s2, s0
-; GFX1150-NEXT: s_mov_b32 s3, s0
+; GFX1150-NEXT: s_mov_b64 s[0:1], 0
+; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_3)
+; GFX1150-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1150-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
; GFX1150-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-NEXT: s_fmamk_f32 s0, s1, 0x40800000, s0
-; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1150-NEXT: v_mov_b32_e32 v0, s0
; GFX1150-NEXT: ; return to shader part epilog
;
+; GIGFX1150-LABEL: _amdgpu_ps_main:
+; GIGFX1150: ; %bb.0: ; %bb
+; GIGFX1150-NEXT: s_mov_b32 s0, 0
+; GIGFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GIGFX1150-NEXT: s_mov_b32 s1, s0
+; GIGFX1150-NEXT: s_mov_b32 s2, s0
+; GIGFX1150-NEXT: s_mov_b32 s3, s0
+; GIGFX1150-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GIGFX1150-NEXT: s_waitcnt lgkmcnt(0)
+; GIGFX1150-NEXT: s_fmamk_f32 s0, s1, 0x40800000, s0
+; GIGFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX1150-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX1150-NEXT: ; return to shader part epilog
+;
; GFX12-LABEL: _amdgpu_ps_main:
; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s1, s0
-; GFX12-NEXT: s_mov_b32 s2, s0
-; GFX12-NEXT: s_mov_b32 s3, s0
+; GFX12-NEXT: s_mov_b64 s[0:1], 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_3)
+; GFX12-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX12-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_fmamk_f32 s0, s1, 0x40800000, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
+;
+; GIGFX12-LABEL: _amdgpu_ps_main:
+; GIGFX12: ; %bb.0: ; %bb
+; GIGFX12-NEXT: s_mov_b32 s0, 0
+; GIGFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GIGFX12-NEXT: s_mov_b32 s1, s0
+; GIGFX12-NEXT: s_mov_b32 s2, s0
+; GIGFX12-NEXT: s_mov_b32 s3, s0
+; GIGFX12-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x0
+; GIGFX12-NEXT: s_wait_kmcnt 0x0
+; GIGFX12-NEXT: s_fmamk_f32 s0, s1, 0x40800000, s0
+; GIGFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GIGFX12-NEXT: v_mov_b32_e32 v0, s0
+; GIGFX12-NEXT: ; return to shader part epilog
bb:
%i = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> zeroinitializer, i32 0, i32 0)
%i1 = bitcast i32 %i to float
diff --git a/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll b/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
index 8cb1d250a6fa7..8f64d9fd283d2 100644
--- a/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
@@ -9,40 +9,34 @@ declare void @llvm.amdgcn.exp.compr.v2f16(i32 immarg, i32 immarg, <2 x half>, <2
define amdgpu_ps void @_amdgpu_ps_main(float %arg) {
; GFX900-LABEL: _amdgpu_ps_main:
; GFX900: ; %bb.0: ; %bb
-; GFX900-NEXT: s_mov_b64 s[4:5], exec
+; GFX900-NEXT: s_mov_b64 s[0:1], exec
; GFX900-NEXT: s_wqm_b64 exec, exec
; GFX900-NEXT: v_mov_b32_e32 v1, v0
-; GFX900-NEXT: s_mov_b32 s0, 0
; GFX900-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v1
; GFX900-NEXT: ; implicit-def: $vgpr0
-; GFX900-NEXT: ; implicit-def: $sgpr2
-; GFX900-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX900-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
+; GFX900-NEXT: ; implicit-def: $sgpr4
+; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX900-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX900-NEXT: s_cbranch_execz .LBB0_2
; GFX900-NEXT: ; %bb.1: ; %bb1
+; GFX900-NEXT: s_mov_b64 s[12:13], 0
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: s_mov_b32 s1, s0
-; GFX900-NEXT: s_mov_b32 s2, s0
-; GFX900-NEXT: s_mov_b32 s3, s0
-; GFX900-NEXT: s_mov_b32 s8, s0
-; GFX900-NEXT: s_mov_b32 s9, s0
-; GFX900-NEXT: s_mov_b32 s10, s0
-; GFX900-NEXT: s_mov_b32 s11, s0
-; GFX900-NEXT: s_mov_b32 s12, s0
-; GFX900-NEXT: s_mov_b32 s13, s0
-; GFX900-NEXT: s_mov_b32 s14, s0
-; GFX900-NEXT: s_mov_b32 s15, s0
-; GFX900-NEXT: image_sample v[0:1], v[0:1], s[8:15], s[0:3] dmask:0x3
-; GFX900-NEXT: s_mov_b32 s2, 1.0
+; GFX900-NEXT: s_mov_b64 s[14:15], s[12:13]
+; GFX900-NEXT: s_mov_b64 s[4:5], s[12:13]
+; GFX900-NEXT: s_mov_b64 s[6:7], s[12:13]
+; GFX900-NEXT: s_mov_b64 s[8:9], s[12:13]
+; GFX900-NEXT: s_mov_b64 s[10:11], s[12:13]
+; GFX900-NEXT: image_sample v[0:1], v[0:1], s[4:11], s[12:15] dmask:0x3
+; GFX900-NEXT: s_mov_b32 s4, 1.0
; GFX900-NEXT: .LBB0_2: ; %Flow
-; GFX900-NEXT: s_or_saveexec_b64 s[0:1], s[6:7]
-; GFX900-NEXT: s_and_b64 exec, exec, s[4:5]
-; GFX900-NEXT: s_and_b64 s[0:1], exec, s[0:1]
-; GFX900-NEXT: v_mov_b32_e32 v2, s2
-; GFX900-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX900-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX900-NEXT: s_and_b64 exec, exec, s[0:1]
+; GFX900-NEXT: s_and_b64 s[2:3], exec, s[2:3]
+; GFX900-NEXT: v_mov_b32_e32 v2, s4
+; GFX900-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execz .LBB0_5
; GFX900-NEXT: ; %bb.3: ; %bb5
-; GFX900-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GFX900-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
; GFX900-NEXT: s_cbranch_scc0 .LBB0_6
; GFX900-NEXT: ; %bb.4: ; %bb5
; GFX900-NEXT: s_mov_b64 exec, 0
@@ -50,7 +44,7 @@ define amdgpu_ps void @_amdgpu_ps_main(float %arg) {
; GFX900-NEXT: v_mov_b32_e32 v1, 0
; GFX900-NEXT: v_mov_b32_e32 v2, 0
; GFX900-NEXT: .LBB0_5: ; %bb6
-; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_cvt_pkrtz_f16_f32 v1, 0, v1
; GFX900-NEXT: v_cvt_pkrtz_f16_f32 v0, v2, v0
diff --git a/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll b/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll
index 5f101c360f148..bb54184dd73c1 100644
--- a/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll
+++ b/llvm/test/CodeGen/AMDGPU/set_kill_i1_for_floation_point_comparison.ll
@@ -6,14 +6,14 @@ define amdgpu_ps void @_amdgpu_ps_main() {
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
+ ; CHECK-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 0
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B]], %subreg.sub0_sub1, [[S_MOV_B]], %subreg.sub2_sub3
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM killed [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; CHECK-NEXT: nofpexcept S_CMP_NLT_F32 [[S_BUFFER_LOAD_DWORD_IMM]], [[S_MOV_B32_1]], implicit-def $scc, implicit $mode
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; CHECK-NEXT: nofpexcept S_CMP_NLT_F32 [[S_BUFFER_LOAD_DWORD_IMM]], [[S_MOV_B32_]], implicit-def $scc, implicit $mode
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY $scc
; CHECK-NEXT: SI_KILL_I1_PSEUDO killed [[COPY]], 0, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; CHECK-NEXT: nofpexcept S_CMP_LT_F32 [[S_BUFFER_LOAD_DWORD_IMM]], [[S_MOV_B32_1]], implicit-def $scc, implicit $mode
+ ; CHECK-NEXT: nofpexcept S_CMP_LT_F32 [[S_BUFFER_LOAD_DWORD_IMM]], [[S_MOV_B32_]], implicit-def $scc, implicit $mode
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.ll b/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.ll
index d34769ad0fcf0..376c49e4ab769 100644
--- a/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-optimize-vgpr-live-range-dbg-instr.ll
@@ -10,9 +10,9 @@ define void @__omp_offloading_35_36570d3__ZN6openmc31process_advance_particle_ev
; GCN-NEXT: .cfi_startproc
; GCN-NEXT: ; %bb.0: ; %bb
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: global_load_dwordx2 v[1:2], v[1:2], off
+; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: v_mov_b32_e32 v4, 0
+; GCN-NEXT: global_load_dwordx2 v[1:2], v[3:4], off
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: s_xor_b64 s[4:5], vcc, -1
@@ -27,8 +27,6 @@ define void @__omp_offloading_35_36570d3__ZN6openmc31process_advance_particle_ev
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
; GCN-NEXT: .LBB0_3: ; %bb2
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: v_mov_b32_e32 v4, v3
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: flat_store_dwordx2 v[1:2], v[3:4]
; GCN-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -36,7 +34,7 @@ define void @__omp_offloading_35_36570d3__ZN6openmc31process_advance_particle_ev
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: .LBB0_4: ; %bb1
; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: v_mov_b32_e32 v4, v3
+; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: flat_store_dwordx2 v[1:2], v[3:4]
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/si-scheduler-exports.ll b/llvm/test/CodeGen/AMDGPU/si-scheduler-exports.ll
index ac271ff6a258b..a2751cad5e902 100644
--- a/llvm/test/CodeGen/AMDGPU/si-scheduler-exports.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-scheduler-exports.ll
@@ -5,15 +5,14 @@ define amdgpu_gs void @_amdgpu_gs_main() {
; CHECK-LABEL: _amdgpu_gs_main:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: s_mov_b32 s0, 0
-; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_mov_b32 s2, s0
-; CHECK-NEXT: v_mov_b32_e32 v1, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_mov_b64 s[2:3], s[0:1]
+; CHECK-NEXT: v_mov_b32_e32 v3, v1
; CHECK-NEXT: v_mov_b32_e32 v2, v0
-; CHECK-NEXT: v_mov_b32_e32 v3, v0
-; CHECK-NEXT: s_mov_b32 s3, s0
; CHECK-NEXT: exp mrt0 off, off, off, off
-; CHECK-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen
+; CHECK-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen
; CHECK-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, i1 false, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll b/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
index be50ad5041692..3808d1157686b 100644
--- a/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
@@ -6,25 +6,22 @@ define protected amdgpu_kernel void @test(ptr addrspace(1) %in, ptr addrspace(1)
; GFX942-LABEL: test:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], 0
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
-; GFX942-NEXT: v_mov_b64_e32 v[10:11], v[2:3]
-; GFX942-NEXT: v_mov_b64_e32 v[8:9], v[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v12, s4
-; GFX942-NEXT: v_mov_b32_e32 v13, s5
-; GFX942-NEXT: v_mov_b32_e32 v4, s6
-; GFX942-NEXT: v_mov_b32_e32 v5, s7
-; GFX942-NEXT: v_mov_b32_e32 v6, s7
-; GFX942-NEXT: v_mov_b32_e32 v7, s7
+; GFX942-NEXT: v_mov_b32_e32 v8, s4
+; GFX942-NEXT: v_mov_b32_e32 v9, s5
+; GFX942-NEXT: v_mov_b32_e32 v0, s6
+; GFX942-NEXT: v_mov_b32_e32 v1, s7
+; GFX942-NEXT: v_mov_b32_e32 v2, s7
+; GFX942-NEXT: v_mov_b32_e32 v3, s7
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[12:13], v[4:7], v13
+; GFX942-NEXT: v_smfmac_i32_16x16x64_i8 v[4:7], v[8:9], v[0:3], v9
; GFX942-NEXT: s_nop 6
-; GFX942-NEXT: global_store_dword v0, v11, s[2:3] offset:12
+; GFX942-NEXT: global_store_dword v10, v7, s[2:3] offset:12
; GFX942-NEXT: s_endpgm
entry:
%arrayidx = getelementptr inbounds i32, ptr addrspace(1) %in, i64 0
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index c611c4b502817..681ceb9e87cbd 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -50,12 +50,12 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: KILL [[S_ADD_U32_]].sub0, [[S_ADD_U32_]].sub1
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_LOAD_DWORDX4_IMM]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = S_MOV_B32 0
+ ; CHECK-NEXT: undef [[S_MOV_B:%[0-9]+]].sub0_sub1:sgpr_128 = S_MOV_B64_IMM_PSEUDO 0
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %118:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+ ; CHECK-NEXT: [[S_MOV_B:%[0-9]+]].sub2_sub3:sgpr_128 = COPY [[S_MOV_B]].sub0_sub1
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %89:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: KILL undef %89:sgpr_128
- ; CHECK-NEXT: KILL undef %118:sgpr_128
; CHECK-NEXT: [[S_SUB_I32_2:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM1]], 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_1:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_1:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_]], implicit-def dead $scc, implicit $scc
@@ -86,19 +86,19 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_U32_11:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %45:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LSHL_B32_]], 16, implicit-def dead $scc
; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LSHL_B32_2]], 16, implicit-def dead $scc
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], undef %301:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B32_]], [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_MOV_B32_]], 16, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %356:sgpr_128, undef %357:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %367:sgpr_128, 16, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B]], [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B]], undef %296:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM [[S_MOV_B]], [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_MOV_B]], 16, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %351:sgpr_128, undef %352:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %362:sgpr_128, 16, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM4:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_3]], 64, 0 :: (invariant load (s128) from %ir.99, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_4]], 64, 0 :: (invariant load (s128) from %ir.107, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM6:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 0, 0 :: (invariant load (s128) from %ir.117, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM7:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_7]], 0, 0 :: (invariant load (s128) from %ir.124, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM2]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %351:sgpr_128, [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %362:sgpr_128, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %346:sgpr_128, [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %357:sgpr_128, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM3]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM]], -98, implicit-def dead $scc
; CHECK-NEXT: [[S_ADD_I32_3:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM1]], -114, implicit-def dead $scc
@@ -116,7 +116,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_LSHL_B32_3:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY12]], 4, implicit-def dead $scc
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN4:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM4]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_ADD_I32_6:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LSHL_B32_3]], 16, implicit-def dead $scc
- ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %383:sgpr_128, [[S_ADD_I32_6]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %378:sgpr_128, [[S_ADD_I32_6]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN5:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM5]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM9:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 224, 0 :: (invariant load (s128) from %ir.129, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM10:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY7]], 224, 0 :: (invariant load (s128) from %ir.145, addrspace 4)
@@ -154,9 +154,10 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN13:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM11]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]].sub3:sgpr_128 = S_MOV_B32 553734060
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]].sub2:sgpr_128 = S_MOV_B32 -1
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
; CHECK-NEXT: [[COPY15:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]]
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM16:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_9]], 0, 0 :: (invariant load (s128) from %ir.170, addrspace 4)
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub1
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]]
; CHECK-NEXT: [[COPY15:%[0-9]+]].sub0:sgpr_128 = COPY [[S_LOAD_DWORD_IMM]]
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[COPY15]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN14:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM15]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
@@ -198,9 +199,9 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[COPY17:%[0-9]+]].sub1:sgpr_128 = COPY [[S_AND_B32_1]]
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[COPY17]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM23:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_16]], 160, 0 :: (invariant load (s128) from %ir.256, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM undef %469:sreg_64, 0, 0 :: (invariant load (s32) from `ptr addrspace(4) undef`, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM undef %464:sreg_64, 0, 0 :: (invariant load (s32) from `ptr addrspace(4) undef`, addrspace 4)
; CHECK-NEXT: KILL [[S_ADD_U32_16]].sub0, [[S_ADD_U32_16]].sub1
- ; CHECK-NEXT: KILL undef %469:sreg_64
+ ; CHECK-NEXT: KILL undef %464:sreg_64
; CHECK-NEXT: KILL [[COPY17]].sub0_sub1_sub2, [[COPY17]].sub3
; CHECK-NEXT: [[S_LSHL_B32_8:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY14]], 3, implicit-def dead $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM24:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_17]], 160, 0 :: (invariant load (s128) from %ir.265, addrspace 4)
@@ -211,8 +212,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_ADD_U32_21]], 168, 0 :: (invariant load (s32) from %ir.305, align 8, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN21:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM23]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN22:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM23]]
; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM24]]
+ ; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM23]]
; CHECK-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_LOAD_DWORD_IMM1]], 65535, implicit-def dead $scc
; CHECK-NEXT: [[COPY18:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]]
; CHECK-NEXT: [[COPY18:%[0-9]+]].sub1:sgpr_128 = COPY [[S_AND_B32_2]]
@@ -351,13 +352,13 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[V_OR_B32_e64_64:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_OR_B32_e64_63]], [[V_ADD_U32_e64_28]], implicit $exec
; CHECK-NEXT: [[V_ADD_U32_e64_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 -593, [[BUFFER_LOAD_FORMAT_X_IDXEN]], 0, implicit $exec
; CHECK-NEXT: [[V_OR_B32_e64_65:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_OR_B32_e64_64]], [[V_ADD_U32_e64_29]], implicit $exec
- ; CHECK-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM undef %542:sreg_64, 0, 0 :: (invariant load (s256) from `ptr addrspace(4) undef`, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM undef %537:sreg_64, 0, 0 :: (invariant load (s256) from `ptr addrspace(4) undef`, addrspace 4)
; CHECK-NEXT: [[V_OR_B32_e64_66:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_OR_B32_e64_65]], [[V_ADD_U32_e64_30]], implicit $exec
; CHECK-NEXT: [[S_ADD_I32_24:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM8]], -594, implicit-def dead $scc
; CHECK-NEXT: [[V_OR_B32_e64_67:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[S_ADD_I32_24]], [[V_OR_B32_e64_66]], implicit $exec
; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 0, [[V_OR_B32_e64_67]], implicit $exec
; CHECK-NEXT: undef [[V_CNDMASK_B32_e64_:%[0-9]+]].sub3:vreg_128 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U32_e64_]], implicit $exec
- ; CHECK-NEXT: IMAGE_STORE_V4_V2_nsa_gfx10 [[V_CNDMASK_B32_e64_]], undef %556:vgpr_32, undef %558:vgpr_32, [[S_LOAD_DWORDX8_IMM]], 15, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
+ ; CHECK-NEXT: IMAGE_STORE_V4_V2_nsa_gfx10 [[V_CNDMASK_B32_e64_]], undef %551:vgpr_32, undef %553:vgpr_32, [[S_LOAD_DWORDX8_IMM]], 15, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
; CHECK-NEXT: S_ENDPGM 0
.expVert:
%0 = extractelement <31 x i32> %userData, i64 2
diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
index dd78c2f46dde8..d160bf71f609c 100644
--- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
@@ -35,11 +35,11 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_load_dwordx4 s[76:79], s[8:9], 0x0
; GLOBALNESS1-NEXT: s_load_dword s6, s[8:9], 0x14
; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v0
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v42, 0
+; GLOBALNESS1-NEXT: v_mov_b32_e32 v40, 0
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS1-NEXT: global_store_dword v[0:1], v42, off
+; GLOBALNESS1-NEXT: global_store_dword v[0:1], v40, off
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT: global_load_dword v2, v42, s[76:77]
+; GLOBALNESS1-NEXT: global_load_dword v2, v40, s[76:77]
; GLOBALNESS1-NEXT: s_mov_b64 s[40:41], s[4:5]
; GLOBALNESS1-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
; GLOBALNESS1-NEXT: s_load_dword s7, s[8:9], 0x20
@@ -79,7 +79,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_mov_b32 s72, s14
; GLOBALNESS1-NEXT: s_mov_b64 s[34:35], s[10:11]
; GLOBALNESS1-NEXT: s_mov_b32 s32, 0
-; GLOBALNESS1-NEXT: ; implicit-def: $vgpr44_vgpr45
+; GLOBALNESS1-NEXT: ; implicit-def: $vgpr42_vgpr43
; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0)
; GLOBALNESS1-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -106,17 +106,17 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: .LBB1_3: ; %Flow28
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GLOBALNESS1-NEXT: v_pk_mov_b32 v[44:45], v[0:1], v[0:1] op_sel:[0,1]
+; GLOBALNESS1-NEXT: v_pk_mov_b32 v[42:43], v[0:1], v[0:1] op_sel:[0,1]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_30
; GLOBALNESS1-NEXT: .LBB1_4: ; %bb5
; GLOBALNESS1-NEXT: ; =>This Loop Header: Depth=1
; GLOBALNESS1-NEXT: ; Child Loop BB1_16 Depth 2
; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0x80
; GLOBALNESS1-NEXT: v_mov_b32_e32 v1, 0
-; GLOBALNESS1-NEXT: flat_load_dword v40, v[0:1]
+; GLOBALNESS1-NEXT: flat_load_dword v44, v[0:1]
; GLOBALNESS1-NEXT: s_add_u32 s8, s38, 40
-; GLOBALNESS1-NEXT: buffer_store_dword v42, off, s[0:3], 0
-; GLOBALNESS1-NEXT: flat_load_dword v46, v[0:1]
+; GLOBALNESS1-NEXT: buffer_store_dword v40, off, s[0:3], 0
+; GLOBALNESS1-NEXT: flat_load_dword v45, v[0:1]
; GLOBALNESS1-NEXT: s_addc_u32 s9, s39, 0
; GLOBALNESS1-NEXT: s_getpc_b64 s[4:5]
; GLOBALNESS1-NEXT: s_add_u32 s4, s4, wobble at gotpcrel32@lo+4
@@ -175,13 +175,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_13
; GLOBALNESS1-NEXT: ; %bb.12: ; %bb39.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], 0, 0
-; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[2:3], off
; GLOBALNESS1-NEXT: .LBB1_13: ; %bb44.lr.ph.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v46
-; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v40, vcc
+; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v45
+; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v44, vcc
; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0)
; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -237,7 +236,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[76:77]
-; GLOBALNESS1-NEXT: v_pk_mov_b32 v[46:47], 0, 0
+; GLOBALNESS1-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[40:41]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[68:69]
@@ -246,14 +245,13 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_mov_b32 s13, s71
; GLOBALNESS1-NEXT: s_mov_b32 s14, s70
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
-; GLOBALNESS1-NEXT: global_store_dwordx2 v[46:47], v[44:45], off
+; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[76:77]
; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[64:65]
; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_14
; GLOBALNESS1-NEXT: ; %bb.23: ; %bb62.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v43, v42
-; GLOBALNESS1-NEXT: global_store_dwordx2 v[46:47], v[42:43], off
+; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[44:45], off
; GLOBALNESS1-NEXT: s_branch .LBB1_14
; GLOBALNESS1-NEXT: .LBB1_24: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1
@@ -273,15 +271,13 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_1
; GLOBALNESS1-NEXT: ; %bb.28: ; %bb69.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], 0, 0
-; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[2:3], off
; GLOBALNESS1-NEXT: s_branch .LBB1_1
; GLOBALNESS1-NEXT: .LBB1_29: ; %bb73.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], 0, 0
-; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[2:3], off
; GLOBALNESS1-NEXT: s_branch .LBB1_2
; GLOBALNESS1-NEXT: .LBB1_30: ; %loop.exit.guard
; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
@@ -327,11 +323,11 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_load_dwordx4 s[72:75], s[8:9], 0x0
; GLOBALNESS0-NEXT: s_load_dword s6, s[8:9], 0x14
; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v0
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v42, 0
+; GLOBALNESS0-NEXT: v_mov_b32_e32 v40, 0
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS0-NEXT: global_store_dword v[0:1], v42, off
+; GLOBALNESS0-NEXT: global_store_dword v[0:1], v40, off
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT: global_load_dword v2, v42, s[72:73]
+; GLOBALNESS0-NEXT: global_load_dword v2, v40, s[72:73]
; GLOBALNESS0-NEXT: s_mov_b64 s[40:41], s[4:5]
; GLOBALNESS0-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
; GLOBALNESS0-NEXT: s_load_dword s7, s[8:9], 0x20
@@ -371,7 +367,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_mov_b32 s70, s14
; GLOBALNESS0-NEXT: s_mov_b64 s[34:35], s[10:11]
; GLOBALNESS0-NEXT: s_mov_b32 s32, 0
-; GLOBALNESS0-NEXT: ; implicit-def: $vgpr44_vgpr45
+; GLOBALNESS0-NEXT: ; implicit-def: $vgpr42_vgpr43
; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0)
; GLOBALNESS0-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -398,17 +394,17 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: .LBB1_3: ; %Flow28
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GLOBALNESS0-NEXT: v_pk_mov_b32 v[44:45], v[0:1], v[0:1] op_sel:[0,1]
+; GLOBALNESS0-NEXT: v_pk_mov_b32 v[42:43], v[0:1], v[0:1] op_sel:[0,1]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_30
; GLOBALNESS0-NEXT: .LBB1_4: ; %bb5
; GLOBALNESS0-NEXT: ; =>This Loop Header: Depth=1
; GLOBALNESS0-NEXT: ; Child Loop BB1_16 Depth 2
; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0x80
; GLOBALNESS0-NEXT: v_mov_b32_e32 v1, 0
-; GLOBALNESS0-NEXT: flat_load_dword v40, v[0:1]
+; GLOBALNESS0-NEXT: flat_load_dword v44, v[0:1]
; GLOBALNESS0-NEXT: s_add_u32 s8, s38, 40
-; GLOBALNESS0-NEXT: buffer_store_dword v42, off, s[0:3], 0
-; GLOBALNESS0-NEXT: flat_load_dword v46, v[0:1]
+; GLOBALNESS0-NEXT: buffer_store_dword v40, off, s[0:3], 0
+; GLOBALNESS0-NEXT: flat_load_dword v45, v[0:1]
; GLOBALNESS0-NEXT: s_addc_u32 s9, s39, 0
; GLOBALNESS0-NEXT: s_getpc_b64 s[4:5]
; GLOBALNESS0-NEXT: s_add_u32 s4, s4, wobble at gotpcrel32@lo+4
@@ -467,13 +463,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_13
; GLOBALNESS0-NEXT: ; %bb.12: ; %bb39.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], 0, 0
-; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[2:3], off
; GLOBALNESS0-NEXT: .LBB1_13: ; %bb44.lr.ph.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v46
-; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v40, vcc
+; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v45
+; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v44, vcc
; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0)
; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -529,7 +524,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[78:79]
-; GLOBALNESS0-NEXT: v_pk_mov_b32 v[46:47], 0, 0
+; GLOBALNESS0-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[40:41]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[72:73]
@@ -538,14 +533,13 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_mov_b32 s13, s69
; GLOBALNESS0-NEXT: s_mov_b32 s14, s68
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
-; GLOBALNESS0-NEXT: global_store_dwordx2 v[46:47], v[44:45], off
+; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[78:79]
; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[64:65]
; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_14
; GLOBALNESS0-NEXT: ; %bb.23: ; %bb62.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v43, v42
-; GLOBALNESS0-NEXT: global_store_dwordx2 v[46:47], v[42:43], off
+; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[44:45], off
; GLOBALNESS0-NEXT: s_branch .LBB1_14
; GLOBALNESS0-NEXT: .LBB1_24: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1
@@ -565,15 +559,13 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_1
; GLOBALNESS0-NEXT: ; %bb.28: ; %bb69.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], 0, 0
-; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[2:3], off
; GLOBALNESS0-NEXT: s_branch .LBB1_1
; GLOBALNESS0-NEXT: .LBB1_29: ; %bb73.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], 0, 0
-; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[42:43], off
+; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[2:3], off
; GLOBALNESS0-NEXT: s_branch .LBB1_2
; GLOBALNESS0-NEXT: .LBB1_30: ; %loop.exit.guard
; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
index b5e4bcd049c42..b5bbb454a34de 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
@@ -464,21 +464,19 @@ define amdgpu_kernel void @v8i8_phi_zeroinit(ptr addrspace(1) %src1, ptr addrspa
; GFX906-NEXT: v_lshlrev_b32_e32 v5, 3, v0
; GFX906-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v0
; GFX906-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GFX906-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
-; GFX906-NEXT: global_load_dwordx2 v[3:4], v5, s[8:9]
+; GFX906-NEXT: global_load_dwordx2 v[1:2], v5, s[8:9]
; GFX906-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX906-NEXT: s_cbranch_execz .LBB9_2
; GFX906-NEXT: ; %bb.1: ; %bb.1
-; GFX906-NEXT: global_load_dwordx2 v[1:2], v5, s[10:11]
-; GFX906-NEXT: s_mov_b32 s4, 0
+; GFX906-NEXT: global_load_dwordx2 v[3:4], v5, s[10:11]
; GFX906-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT: s_mov_b32 s5, s4
; GFX906-NEXT: s_waitcnt vmcnt(1)
-; GFX906-NEXT: v_mov_b32_e32 v3, s4
-; GFX906-NEXT: v_mov_b32_e32 v4, s5
+; GFX906-NEXT: v_mov_b32_e32 v1, 0
; GFX906-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
; GFX906-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX906-NEXT: v_mov_b32_e32 v2, 0
; GFX906-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX906-NEXT: .LBB9_2: ; %Flow
; GFX906-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -486,15 +484,15 @@ define amdgpu_kernel void @v8i8_phi_zeroinit(ptr addrspace(1) %src1, ptr addrspa
; GFX906-NEXT: s_cbranch_execz .LBB9_4
; GFX906-NEXT: ; %bb.3: ; %bb.2
; GFX906-NEXT: s_waitcnt vmcnt(0)
-; GFX906-NEXT: v_mov_b32_e32 v1, v3
+; GFX906-NEXT: v_mov_b32_e32 v4, v2
; GFX906-NEXT: v_mov_b32_e32 v0, 0
-; GFX906-NEXT: v_mov_b32_e32 v2, v4
-; GFX906-NEXT: global_store_dwordx2 v0, v[3:4], s[12:13]
+; GFX906-NEXT: v_mov_b32_e32 v3, v1
+; GFX906-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
; GFX906-NEXT: .LBB9_4: ; %bb.3
; GFX906-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt vmcnt(0)
-; GFX906-NEXT: global_store_dwordx2 v0, v[1:2], s[14:15]
+; GFX906-NEXT: global_store_dwordx2 v0, v[3:4], s[14:15]
; GFX906-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/vopc_dpp.ll b/llvm/test/CodeGen/AMDGPU/vopc_dpp.ll
index a6dcbb5bbd695..cec401c0a9713 100644
--- a/llvm/test/CodeGen/AMDGPU/vopc_dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/vopc_dpp.ll
@@ -2,7 +2,7 @@
define amdgpu_cs void @_amdgpu_cs_main(i32 %0) {
; GFX11-LABEL: _amdgpu_cs_main:
-; GFX11: v_cmp_eq_u32_e64_dpp s1, v1, v0 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX11: v_cmp_eq_u32_e64_dpp s0, v1, v0 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
.entry:
%1 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 0, i32 0, i32 15, i32 15, i1 false)
%2 = icmp ne i32 %1, %0
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll b/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
index 6133cb4690723..53172b7ce7ca7 100644
--- a/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
+++ b/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
@@ -30,7 +30,8 @@ define amdgpu_kernel void @foo(i1 %cmp1) {
; GFX906-NEXT: buffer_load_dword v6, off, s[12:15], 0 offset:12
; GFX906-NEXT: s_load_dword s2, s[4:5], 0x24
; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x1c
-; GFX906-NEXT: s_mov_b32 s4, 0
+; GFX906-NEXT: s_mov_b64 s[4:5], 0
+; GFX906-NEXT: s_mov_b64 s[6:7], exec
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: s_bitcmp1_b32 s2, 0
; GFX906-NEXT: s_mul_i32 s0, s0, s1
@@ -38,9 +39,9 @@ define amdgpu_kernel void @foo(i1 %cmp1) {
; GFX906-NEXT: v_mad_u32_u24 v0, s0, v0, v1
; GFX906-NEXT: v_add_lshl_u32 v2, v0, v2, 4
; GFX906-NEXT: v_mov_b32_e32 v0, 0
-; GFX906-NEXT: v_mov_b32_e32 v1, v0
-; GFX906-NEXT: s_cselect_b32 s5, 1, 0
-; GFX906-NEXT: s_mov_b64 s[2:3], exec
+; GFX906-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-NEXT: s_mov_b32 s2, 0
+; GFX906-NEXT: s_cselect_b32 s3, 1, 0
; GFX906-NEXT: ds_write_b64 v2, v[0:1]
; GFX906-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX906-NEXT: s_waitcnt vmcnt(3)
@@ -59,13 +60,11 @@ define amdgpu_kernel void @foo(i1 %cmp1) {
; GFX906-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX906-NEXT: s_cbranch_execnz .LBB0_1
; GFX906-NEXT: ; %bb.2:
-; GFX906-NEXT: s_cmp_lg_u32 s5, 0
-; GFX906-NEXT: s_mov_b64 exec, s[2:3]
-; GFX906-NEXT: s_cselect_b32 s5, 0x3ff00000, 0
-; GFX906-NEXT: v_cvt_f32_f64_e32 v0, s[4:5]
-; GFX906-NEXT: s_mov_b32 s5, s4
-; GFX906-NEXT: s_mov_b32 s6, s4
-; GFX906-NEXT: s_mov_b32 s7, s4
+; GFX906-NEXT: s_cmp_lg_u32 s3, 0
+; GFX906-NEXT: s_mov_b64 exec, s[6:7]
+; GFX906-NEXT: s_cselect_b32 s3, 0x3ff00000, 0
+; GFX906-NEXT: v_cvt_f32_f64_e32 v0, s[2:3]
+; GFX906-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX906-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX906-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index deab407581880..360e987e2b43b 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -3398,38 +3398,38 @@ define amdgpu_gs void @wqm_init_exec() {
; GFX9-W64-LABEL: wqm_init_exec:
; GFX9-W64: ; %bb.0: ; %bb
; GFX9-W64-NEXT: s_mov_b64 exec, -1
-; GFX9-W64-NEXT: s_mov_b32 s0, 0
; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-W64-NEXT: s_mov_b32 s1, s0
-; GFX9-W64-NEXT: s_mov_b32 s2, s0
-; GFX9-W64-NEXT: s_mov_b32 s3, s0
-; GFX9-W64-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-W64-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-W64-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-W64-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-W64-NEXT: s_mov_b32 s4, 0
; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX9-W64-NEXT: s_wqm_b64 exec, exec
-; GFX9-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $exec
-; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $exec
+; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-W64-NEXT: v_mov_b32_e32 v1, s4
; GFX9-W64-NEXT: ds_write_b32 v0, v1
; GFX9-W64-NEXT: s_endpgm
;
; GFX10-W32-LABEL: wqm_init_exec:
; GFX10-W32: ; %bb.0: ; %bb
; GFX10-W32-NEXT: s_mov_b32 exec_lo, -1
-; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-W32-NEXT: s_mov_b32 s0, 0
+; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: s_mov_b32 s2, 0
; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10-W32-NEXT: s_mov_b32 s2, s0
-; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s1
-; GFX10-W32-NEXT: v_mov_b32_e32 v1, v0
+; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX10-W32-NEXT: v_mov_b32_e32 v3, v1
; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-W32-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-W32-NEXT: v_mov_b32_e32 v4, s0
-; GFX10-W32-NEXT: s_mov_b32 s1, s0
-; GFX10-W32-NEXT: s_mov_b32 s3, s0
+; GFX10-W32-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v5, s2
+; GFX10-W32-NEXT: s_mov_b64 s[0:1], 0
+; GFX10-W32-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; GFX10-W32-NEXT: ds_write_b32 v0, v4
+; GFX10-W32-NEXT: ds_write_b32 v4, v5
; GFX10-W32-NEXT: s_endpgm
bb:
call void @llvm.amdgcn.init.exec(i64 -1)
More information about the llvm-commits
mailing list