[llvm-branch-commits] [llvm] AMDGPU: si-fold-operands various improvements (PR #216074)
Petar Avramovic via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 13 08:08:53 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/216074
>From bc10ee220c5d6c63c9c668af18e6b5ef98bf408c Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Thu, 13 Aug 2026 16:12:51 +0200
Subject: [PATCH] AMDGPU: si-fold-operands various improvements
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 180 ++-
llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll | 28 +-
.../AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll | 98 +-
.../AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll | 130 +-
.../GlobalISel/constant-bus-restriction.ll | 12 +-
.../AMDGPU/GlobalISel/cvt_f32_ubyte.ll | 7 +-
...vergence-divergent-i1-used-outside-loop.ll | 20 +-
.../divergence-temporal-divergent-i1.ll | 46 +-
.../AMDGPU/GlobalISel/extractelement.i128.ll | 37 +-
.../AMDGPU/GlobalISel/extractelement.i16.ll | 74 +-
.../AMDGPU/GlobalISel/extractelement.i8.ll | 216 ++-
.../AMDGPU/GlobalISel/extractelement.ll | 87 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll | 19 +-
.../CodeGen/AMDGPU/GlobalISel/floor.f64.ll | 24 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll | 19 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll | 86 +-
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 490 +++----
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 369 +++---
.../AMDGPU/GlobalISel/insertelement.i16.ll | 652 +++++----
.../AMDGPU/GlobalISel/insertelement.i8.ll | 389 +++---
.../AMDGPU/GlobalISel/insertelement.ll | 650 +++------
.../AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll | 5 +-
.../GlobalISel/llvm.amdgcn.div.scale.ll | 34 +-
.../GlobalISel/llvm.amdgcn.intersect_ray.ll | 86 +-
.../CodeGen/AMDGPU/GlobalISel/mubuf-global.ll | 109 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 358 +++--
llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll | 28 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 587 +++++----
.../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 90 +-
.../AMDGPU/GlobalISel/shl-ext-reduce.ll | 25 +-
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 555 ++++----
.../CodeGen/AMDGPU/GlobalISel/udiv.i64.ll | 82 +-
.../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 54 +-
.../CodeGen/AMDGPU/GlobalISel/urem.i64.ll | 82 +-
llvm/test/CodeGen/AMDGPU/bfi_int.ll | 100 +-
llvm/test/CodeGen/AMDGPU/ctpop.ll | 10 +-
llvm/test/CodeGen/AMDGPU/ctpop16.ll | 10 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 8 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 1085 +++++++--------
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 25 +-
llvm/test/CodeGen/AMDGPU/fneg-fabs-v2f32.ll | 18 +-
llvm/test/CodeGen/AMDGPU/fold-readlane.mir | 8 +-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll | 81 +-
llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll | 56 +-
.../AMDGPU/gfx12_scalar_subword_loads.ll | 18 +-
llvm/test/CodeGen/AMDGPU/global-saddr-load.ll | 102 +-
.../AMDGPU/llvm.amdgcn.av.load.b128.ll | 378 +++---
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll | 3 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll | 3 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll | 15 +-
.../llvm.amdgcn.global.load.async.to.lds.ll | 35 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 48 +-
.../CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll | 695 +++-------
.../AMDGPU/llvm.amdgcn.intersect_ray.ll | 60 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll | 57 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll | 57 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 78 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 100 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll | 28 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 246 +---
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 48 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll | 16 +-
.../CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll | 9 +-
.../AMDGPU/llvm.amdgcn.sched.group.barrier.ll | 1173 +++++------------
llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll | 150 +--
llvm/test/CodeGen/AMDGPU/llvm.exp.ll | 4 +-
llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll | 150 +--
llvm/test/CodeGen/AMDGPU/llvm.exp10.ll | 4 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll | 344 +++--
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll | 362 ++---
llvm/test/CodeGen/AMDGPU/llvm.log.ll | 660 +++++-----
llvm/test/CodeGen/AMDGPU/llvm.log10.ll | 660 +++++-----
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 310 ++---
...alizer-lds-dma-volatile-and-nontemporal.ll | 36 +-
.../AMDGPU/memset-param-combinations.ll | 88 +-
.../move-addr64-rsrc-dead-subreg-writes.ll | 6 +-
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 30 +-
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 41 +-
.../AMDGPU/pseudo-scalar-transcendental.ll | 63 +-
llvm/test/CodeGen/AMDGPU/rsq.f64.ll | 202 +--
.../CodeGen/AMDGPU/scale-offset-global.ll | 38 +-
83 files changed, 5793 insertions(+), 7669 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 7401ff8456cae..5c0551f421e33 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -90,6 +90,11 @@ struct FoldableDef {
return OpToFold->getSubReg();
}
+ /// Subregister the use has to read to get the value of this def.
+ unsigned getUseSubReg(const SIRegisterInfo &TRI) const {
+ return TRI.composeSubRegIndices(getSubReg(), DefSubReg);
+ }
+
bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
bool isFI() const {
@@ -133,10 +138,16 @@ struct FoldableDef {
return TII.isOperandLegal(MI, OpIdx, &TmpOp);
}
default:
- // TODO: Try to apply DefSubReg, for global address we can extract
- // low/high.
- if (DefSubReg != AMDGPU::NoSubRegister)
- return false;
+ if (DefSubReg != AMDGPU::NoSubRegister) {
+ // TODO: For a global address we can extract low/high.
+ if (!OpToFold->isReg() || OpToFold->getSubReg())
+ return false;
+ // The copy keeps the parent instruction, so the operand still knows
+ // which function it belongs to.
+ MachineOperand SubRegOp = *OpToFold;
+ SubRegOp.setSubReg(DefSubReg);
+ return TII.isOperandLegal(MI, OpIdx, &SubRegOp);
+ }
return TII.isOperandLegal(MI, OpIdx, OpToFold);
}
@@ -253,7 +264,10 @@ class SIFoldOperandsImpl {
bool tryFoldZeroHighBits(MachineInstr &MI) const;
bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
+ bool blocksBetterSGPRFold(const MachineInstr &Copy) const;
+
bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
+ bool foldCopyToVGPRRegSequenceOfImm(MachineInstr *CopyMI) const;
bool tryFoldFoldableCopy(MachineInstr &MI,
MachineOperand *&CurrentKnownM0Val) const;
@@ -748,16 +762,28 @@ bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
MachineOperand *New = Fold.Def.OpToFold;
+ if (Old.getSubReg()) {
+ // Legality was checked when the candidate was added, but other operands of
+ // the instruction may have been folded since then, so the constant bus
+ // limit has to be checked against the current instruction.
+ MachineOperand Tmp = *New;
+ Tmp.setSubReg(TRI->composeSubRegIndices(New->getSubReg(), Old.getSubReg()));
+ if (!TII->isOperandLegal(*MI, Fold.UseOpNo, &Tmp))
+ return false;
+ }
+
// Verify the register is compatible with the operand.
if (const TargetRegisterClass *OpRC =
TII->getRegClass(MI->getDesc(), Fold.UseOpNo)) {
const TargetRegisterClass *NewRC =
TRI->getRegClassForReg(*MRI, New->getReg());
+ unsigned SubReg =
+ TRI->composeSubRegIndices(New->getSubReg(), Old.getSubReg());
+
const TargetRegisterClass *ConstrainRC = OpRC;
- if (New->getSubReg()) {
- ConstrainRC =
- TRI->getMatchingSuperRegClass(NewRC, OpRC, New->getSubReg());
+ if (SubReg) {
+ ConstrainRC = TRI->getMatchingSuperRegClass(NewRC, OpRC, SubReg);
if (!ConstrainRC)
return false;
@@ -1233,7 +1259,9 @@ bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
return false;
}
- if (!TRI->getCommonSubClass(OpRC, SplatRC))
+ // The operand is replaced by an immediate, so the register banks do not
+ // have to match. Only the element size has to.
+ if (TRI->getRegSizeInBits(*OpRC) != TRI->getRegSizeInBits(*SplatRC))
return false;
}
@@ -1279,15 +1307,19 @@ bool SIFoldOperandsImpl::foldOperand(
if (isTemporallyDivergentUse(OpToFold, *UseMI))
return Changed;
- // FIXME: Fold operands with subregs.
if (UseOp->isReg() && OpToFold.isReg()) {
if (UseOp->isImplicit())
return Changed;
// Allow folding from SGPRs to 16-bit VGPRs.
if (UseOp->getSubReg() != AMDGPU::NoSubRegister &&
(UseOp->getSubReg() != AMDGPU::lo16 ||
- !TRI->isSGPRReg(*MRI, OpToFold.getReg())))
- return Changed;
+ !TRI->isSGPRReg(*MRI, OpToFold.getReg()))) {
+ // The use reads a subregister of the folded value, which is already
+ // applied to DefSubReg by the caller. Composing it with a subregister on
+ // the def side is not handled.
+ if (OpToFold.getSubReg())
+ return Changed;
+ }
}
// Special case for REG_SEQUENCE: We can't fold literals into
@@ -1573,7 +1605,7 @@ bool SIFoldOperandsImpl::foldOperand(
// %sgpr1 = COPY %sgpr0
UseMI->setDesc(TII->get(AMDGPU::COPY));
UseMI->getOperand(1).setReg(OpToFold.getReg());
- UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
+ UseMI->getOperand(1).setSubReg(OpToFold.getUseSubReg(*TRI));
UseMI->getOperand(1).setIsKill(false);
UseMI->removeOperand(2); // Remove exec read (or src1 for readlane)
UseMI->clearFlag(MachineInstr::NoConvergent);
@@ -2077,6 +2109,125 @@ bool SIFoldOperandsImpl::foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const {
return true;
}
+/// Fold %vgpr = COPY (REG_SEQUENCE of scalar immediates) into a REG_SEQUENCE of
+/// 32 bit VGPR immediate moves, so that the immediates can be folded into the
+/// vector uses instead of being built in SGPRs and copied over.
+bool SIFoldOperandsImpl::foldCopyToVGPRRegSequenceOfImm(
+ MachineInstr *CopyMI) const {
+ const TargetRegisterClass *DstRC =
+ MRI->getRegClass(CopyMI->getOperand(0).getReg());
+ if (!TRI->isVGPRClass(DstRC) || DstRC->getSizeInBits() <= 32)
+ return false;
+
+ Register UseReg = CopyMI->getOperand(1).getReg();
+ if (!TRI->isSGPRReg(*MRI, UseReg) || !MRI->hasOneNonDBGUse(UseReg))
+ return false;
+
+ // Only worth it if the immediates can fold into the uses. A use of the whole
+ // tuple could have taken the scalar registers directly instead.
+ for (const MachineOperand &UseMO :
+ MRI->use_nodbg_operands(CopyMI->getOperand(0).getReg())) {
+ unsigned UseSubIdx = UseMO.getSubReg();
+ if (!UseSubIdx || TRI->getSubRegIdxSize(UseSubIdx) != 32)
+ return false;
+ }
+
+ MachineInstr *RegSeq = MRI->getVRegDef(UseReg);
+ if (!RegSeq || !RegSeq->isRegSequence())
+ return false;
+
+ // Immediate value and the 32 bit subregister of the destination it defines.
+ SmallVector<std::pair<int64_t, unsigned>, 16> Lanes;
+ for (unsigned I = 1, E = RegSeq->getNumOperands(); I != E; I += 2) {
+ MachineOperand &RegOp = RegSeq->getOperand(I);
+ unsigned SubIdx = RegSeq->getOperand(I + 1).getImm();
+ // Rematerializing a value that is live anyway only adds moves.
+ if (RegOp.getSubReg() || !MRI->hasOneNonDBGUse(RegOp.getReg()))
+ return false;
+
+ MachineOperand *Lookup = lookUpCopyChain(*TII, *MRI, RegOp.getReg());
+ if (!Lookup || !Lookup->isImm())
+ return false;
+
+ unsigned Size = TRI->getSubRegIdxSize(SubIdx);
+ if (Size == 32) {
+ Lanes.emplace_back(Lookup->getImm(), SubIdx);
+ } else if (Size == 64 && !ST->hasVMovB64Inst()) {
+ // Without a 64 bit move the value is materialized in halves anyway, and
+ // 32 bit immediates can fold into more uses.
+ uint64_t Imm = Lookup->getImm();
+ Lanes.emplace_back(static_cast<int32_t>(Imm),
+ TRI->composeSubRegIndices(SubIdx, AMDGPU::sub0));
+ Lanes.emplace_back(static_cast<int32_t>(Imm >> 32),
+ TRI->composeSubRegIndices(SubIdx, AMDGPU::sub1));
+ } else {
+ return false;
+ }
+ }
+
+ if (Lanes.size() * 32 != DstRC->getSizeInBits())
+ return false;
+
+ for (auto [Imm, SubIdx] : Lanes) {
+ if (!TRI->getSubRegisterClass(DstRC, SubIdx))
+ return false;
+ }
+
+ const DebugLoc &DL = CopyMI->getDebugLoc();
+ MachineBasicBlock &MBB = *CopyMI->getParent();
+ MachineInstrBuilder B(*MBB.getParent(), CopyMI);
+
+ CopyMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE));
+ for (unsigned I = CopyMI->getNumOperands() - 1; I > 0; --I)
+ CopyMI->removeOperand(I);
+
+ for (auto [Imm, SubIdx] : Lanes) {
+ Register Tmp = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, CopyMI, DL, TII->get(AMDGPU::V_MOV_B32_e32), Tmp).addImm(Imm);
+ B.addReg(Tmp).addImm(SubIdx);
+ }
+
+ LLVM_DEBUG(dbgs() << "Folded " << *CopyMI);
+ return true;
+}
+
+// Folding a copy from an SGPR into a VALU use spends the constant bus of that
+// use. If a source of the same use is a copy from an SGPR with fewer uses,
+// folding this copy blocks more copies than it removes.
+bool SIFoldOperandsImpl::blocksBetterSGPRFold(const MachineInstr &Copy) const {
+ const MachineOperand &SrcMO = Copy.getOperand(1);
+ Register Dst = Copy.getOperand(0).getReg();
+ if (!SrcMO.isReg() || !SrcMO.getReg().isVirtual() || !Dst.isVirtual() ||
+ !TRI->isSGPRReg(*MRI, SrcMO.getReg()) || !TRI->isVGPR(*MRI, Dst))
+ return false;
+
+ unsigned NumUses = std::distance(MRI->use_nodbg_operands(Dst).begin(),
+ MRI->use_nodbg_operands(Dst).end());
+ if (NumUses < 2)
+ return false;
+
+ for (const MachineInstr &UseMI : MRI->use_nodbg_instructions(Dst)) {
+ if (!SIInstrInfo::isVALU(UseMI, /*AllowLDSDMA=*/false))
+ continue;
+ for (const MachineOperand &MO : UseMI.explicit_uses()) {
+ if (!MO.isReg() || !MO.getReg().isVirtual() || MO.getReg() == Dst)
+ continue;
+ const MachineInstr *Def = MRI->getUniqueVRegDef(MO.getReg());
+ if (!Def || !Def->isCopy() || Def->getOperand(1).getSubReg())
+ continue;
+ const MachineOperand &OtherSrc = Def->getOperand(1);
+ if (!OtherSrc.isReg() || !OtherSrc.getReg().isVirtual() ||
+ !TRI->isSGPRReg(*MRI, OtherSrc.getReg()))
+ continue;
+ if (std::distance(MRI->use_nodbg_operands(MO.getReg()).begin(),
+ MRI->use_nodbg_operands(MO.getReg()).end()) < NumUses)
+ return true;
+ }
+ }
+
+ return false;
+}
+
bool SIFoldOperandsImpl::tryFoldFoldableCopy(
MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
Register DstReg = MI.getOperand(0).getReg();
@@ -2151,11 +2302,16 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
}
}
+ if (MI.isCopy() && blocksBetterSGPRFold(MI))
+ return false;
+
// Fold copy to AGPR through reg_sequence
// TODO: Handle with subregister extract
if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(1).getSubReg()) {
if (foldCopyToAGPRRegSequence(&MI))
return true;
+ if (foldCopyToVGPRRegSequenceOfImm(&MI))
+ return true;
}
FoldableDef Def(OpToFold, DstRC);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
index d553f16d49013..366fbf9b814a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
@@ -291,28 +291,16 @@ define amdgpu_ps <2 x float> @v_andn2_i64_vs(i64 %src0, i64 inreg %src1) {
; GCN-LABEL: v_andn2_i64_vs:
; GCN: ; %bb.0:
; GCN-NEXT: s_not_b64 s[0:1], s[2:3]
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_and_b32_e32 v0, v0, v2
-; GCN-NEXT: v_and_b32_e32 v1, v1, v3
+; GCN-NEXT: v_and_b32_e32 v0, s0, v0
+; GCN-NEXT: v_and_b32_e32 v1, s1, v1
; GCN-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: v_andn2_i64_vs:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_andn2_i64_vs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: v_andn2_i64_vs:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_not_b64 s[0:1], s[2:3]
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX10PLUS-NEXT: v_and_b32_e32 v1, s1, v1
+; GFX10PLUS-NEXT: ; return to shader part epilog
%not.src1 = xor i64 %src1, -1
%and = and i64 %src0, %not.src1
%cast = bitcast i64 %and to <2 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
index e74012ef6f689..16c9c29a9b9b2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
@@ -855,14 +855,13 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa
; CI-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -875,14 +874,13 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa
; VI-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -1609,14 +1607,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
; CI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -1629,14 +1626,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
; VI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -1649,15 +1645,14 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1670,13 +1665,11 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX10-NEXT: v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: flat_atomic_dec v[0:1], v2
@@ -1689,15 +1682,14 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
; GFX11-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v2, 42
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_u32 v0, s0, s0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v2, 42
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:20
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -2338,14 +2330,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
; CI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v0, 42
@@ -2359,14 +2350,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
; VI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v0, 42
@@ -2385,9 +2375,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v0
+; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, s0, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX9-NEXT: flat_atomic_dec_x2 v[3:4], v[1:2] offset:40
@@ -2402,15 +2391,13 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v2, s0, s0, v0
+; GFX10-NEXT: v_add_co_ci_u32_e64 v3, s0, s1, 0, s0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -2422,16 +2409,15 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
; GFX11-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: v_dual_mov_b32 v2, 42 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 3, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-NEXT: v_add_co_u32 v2, s0, s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] offset:40
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, 0, s0
+; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] offset:40
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -3438,14 +3424,13 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa
; CI-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v0, 42
@@ -3459,14 +3444,13 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa
; VI-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v0, 42
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
index 9f4d4469af19c..beafb94e46961 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
@@ -980,14 +980,13 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa
; CI-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -1000,14 +999,13 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa
; VI-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -2217,14 +2215,13 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa
; CI-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v0, 42
@@ -2238,14 +2235,13 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa
; VI-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v0, 42
@@ -3087,14 +3083,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
; CI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -3107,14 +3102,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
; VI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
@@ -3127,15 +3121,14 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -3148,13 +3141,11 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX10-NEXT: v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: flat_atomic_inc v[0:1], v2
@@ -3167,15 +3158,14 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v2, 42
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT: v_add_co_u32 v0, s0, s0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v2, 42
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:20
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -3186,15 +3176,14 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
; GFX12-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: v_mov_b32_e32 v2, 42
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX12-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:20 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
@@ -4165,14 +4154,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
; CI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
+; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v0, 42
@@ -4186,14 +4174,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
; VI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v0, 42
@@ -4212,9 +4199,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v0
+; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, s0, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX9-NEXT: flat_atomic_inc_x2 v[3:4], v[1:2] offset:40
@@ -4229,15 +4215,13 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v2, s0, s0, v0
+; GFX10-NEXT: v_add_co_ci_u32_e64 v3, s0, s1, 0, s0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -4249,16 +4233,15 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: v_dual_mov_b32 v2, 42 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 3, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX11-NEXT: v_add_co_u32 v2, s0, s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:40
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, 0, s0
+; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:40
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -4268,16 +4251,15 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
; GFX12-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, 42 :: v_dual_mov_b32 v3, 0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-NEXT: v_mov_b32_e32 v0, 42
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 3, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:40 scope:SCOPE_DEV
+; GFX12-NEXT: v_add_co_u32 v2, s0, s0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, s1, 0, s0
+; GFX12-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:40 scope:SCOPE_DEV
; GFX12-NEXT: s_wait_storecnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll
index 1ba09ddd21dda..821c094dada6c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll
@@ -71,8 +71,8 @@ define amdgpu_ps float @fma_sss(float inreg %src) {
define amdgpu_ps float @fma_ss_s(float inreg %src01, float inreg %src2) {
; GFX9-LABEL: fma_ss_s:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_fma_f32 v0, s2, s2, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_fma_f32 v0, v0, v0, s3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: fma_ss_s:
@@ -103,8 +103,8 @@ define amdgpu_ps float @fma_s_ss(float inreg %src0, float inreg %src12) {
define amdgpu_ps float @fma_ss_s_same_outer(float inreg %src02, float inreg %src1) {
; GFX9-LABEL: fma_ss_s_same_outer:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_fma_f32 v0, s2, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_fma_f32 v0, v0, s3, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: fma_ss_s_same_outer:
@@ -223,8 +223,8 @@ define amdgpu_ps float @class_s_s(float inreg %src0, i32 inreg %src1) {
define amdgpu_ps float @div_scale_s_s_true(float inreg %src0, float inreg %src1) {
; GFX9-LABEL: div_scale_s_s_true:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_div_scale_f32 v0, s[0:1], s2, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_div_scale_f32 v0, s[0:1], v0, s3, v0
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: div_scale_s_s_true:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
index 7bd7196c5241b..2abe02d2766df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
@@ -770,17 +770,16 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
; VI-LABEL: load_v4i8_to_v4f32_2_uses:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT: v_mov_b32_e32 v2, 0xff
; VI-NEXT: v_mov_b32_e32 v6, 9
; VI-NEXT: v_mov_b32_e32 v7, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v1, v[0:1]
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v2, 0xff
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: v_mov_b32_e32 v4, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
index ac93b626bce90..6d8ce378b07a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -268,15 +268,13 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
; GFX10-NEXT: s_andn2_b32 s10, s10, exec_lo
; GFX10-NEXT: s_lshl_b64 s[12:13], s[4:5], 2
; GFX10-NEXT: s_andn2_b32 s5, s11, exec_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s12
-; GFX10-NEXT: v_mov_b32_e32 v6, s13
+; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, s12
+; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s13, v2, vcc_lo
; GFX10-NEXT: s_and_b32 s11, exec_lo, exec_lo
; GFX10-NEXT: s_and_b32 s12, exec_lo, exec_lo
; GFX10-NEXT: s_or_b32 s11, s5, s11
-; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, v5
-; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
-; GFX10-NEXT: s_or_b32 s10, s10, s12
; GFX10-NEXT: global_load_dword v5, v[5:6], off
+; GFX10-NEXT: s_or_b32 s10, s10, s12
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v5
; GFX10-NEXT: s_and_saveexec_b32 s5, vcc_lo
@@ -372,10 +370,8 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
; GFX10-NEXT: s_ashr_i32 s5, s4, 31
; GFX10-NEXT: v_mov_b32_e32 v5, s4
; GFX10-NEXT: s_lshl_b64 s[10:11], s[4:5], 2
-; GFX10-NEXT: v_mov_b32_e32 v8, s10
-; GFX10-NEXT: v_mov_b32_e32 v9, s11
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, v8
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, s10
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s11, v3, vcc_lo
; GFX10-NEXT: global_store_dword v[8:9], v5, off
; GFX10-NEXT: .LBB5_4: ; %loop.break.block
; GFX10-NEXT: ; in Loop: Header=BB5_2 Depth=1
@@ -489,10 +485,8 @@ define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspa
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
; GFX10-NEXT: s_lshl_b64 s[6:7], s[0:1], 2
; GFX10-NEXT: s_andn2_b32 s1, s4, exec_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s6
-; GFX10-NEXT: v_mov_b32_e32 v6, s7
-; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, v5
-; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v1, s6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, s7, v2, vcc_lo
; GFX10-NEXT: global_load_dword v5, v[5:6], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
index 8c344daa5c38e..1804e6ac59363 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
@@ -108,15 +108,13 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr ad
; GFX10-NEXT: s_branch .LBB2_3
; GFX10-NEXT: .LBB2_1: ; %loop.body
; GFX10-NEXT: ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, s6
-; GFX10-NEXT: v_mov_b32_e32 v5, s7
-; GFX10-NEXT: s_andn2_b32 s6, s10, exec_lo
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, s6
+; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s7, v1, vcc_lo
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s4, v2
+; GFX10-NEXT: s_andn2_b32 s6, s10, exec_lo
; GFX10-NEXT: s_add_i32 s4, s4, 1
; GFX10-NEXT: global_load_dword v6, v[4:5], off
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_and_b32 s7, exec_lo, vcc_lo
; GFX10-NEXT: s_or_b32 s10, s6, s7
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -207,12 +205,10 @@ define void @nested_loops_temporal_divergence_inner(float %pre.cond.val, i32 %n.
; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
; GFX10-NEXT: ; implicit-def: $sgpr9
-; GFX10-NEXT: v_mov_b32_e32 v6, s10
-; GFX10-NEXT: v_mov_b32_e32 v7, s11
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s10
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s11, v3, vcc_lo
; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[6:7]
; GFX10-NEXT: .LBB3_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB3_1 Depth=1
@@ -232,13 +228,11 @@ define void @nested_loops_temporal_divergence_inner(float %pre.cond.val, i32 %n.
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB3_1 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
+; GFX10-NEXT: v_add_co_u32 v6, s4, v4, s6
+; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, s7, v5, s4
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: v_add_co_u32 v6, s4, v4, v6
-; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: flat_store_byte v[6:7], v0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -298,12 +292,10 @@ define void @nested_loops_temporal_divergence_outer(float %pre.cond.val, i32 %n.
; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
; GFX10-NEXT: ; implicit-def: $sgpr9
-; GFX10-NEXT: v_mov_b32_e32 v6, s10
-; GFX10-NEXT: v_mov_b32_e32 v7, s11
+; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, s10
+; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, s11, v3, vcc_lo
; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
-; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v2, v6
-; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[6:7]
; GFX10-NEXT: .LBB4_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB4_1 Depth=1
@@ -323,13 +315,11 @@ define void @nested_loops_temporal_divergence_outer(float %pre.cond.val, i32 %n.
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB4_1 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
+; GFX10-NEXT: v_add_co_u32 v6, s4, v4, s6
+; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, s7, v5, s4
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: v_add_co_u32 v6, s4, v4, v6
-; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: flat_store_byte v[6:7], v0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -389,12 +379,10 @@ define void @nested_loops_temporal_divergence_both(float %pre.cond.val, i32 %n.i
; GFX10-NEXT: s_ashr_i32 s7, s6, 31
; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: s_lshl_b64 s[10:11], s[6:7], 2
-; GFX10-NEXT: v_mov_b32_e32 v8, s10
-; GFX10-NEXT: v_mov_b32_e32 v9, s11
+; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, s10
+; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, s11, v3, vcc_lo
; GFX10-NEXT: s_mov_b32 s10, 0
; GFX10-NEXT: s_mov_b32 s11, 0
-; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v2, v8
-; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
; GFX10-NEXT: flat_load_dword v0, v[8:9]
; GFX10-NEXT: .LBB5_2: ; %InnerHeader
; GFX10-NEXT: ; Parent Loop BB5_1 Depth=1
@@ -414,13 +402,11 @@ define void @nested_loops_temporal_divergence_both(float %pre.cond.val, i32 %n.i
; GFX10-NEXT: ; %bb.3: ; %UseInst
; GFX10-NEXT: ; in Loop: Header=BB5_1 Depth=1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX10-NEXT: v_mov_b32_e32 v8, s6
-; GFX10-NEXT: v_mov_b32_e32 v9, s7
; GFX10-NEXT: v_cmp_lt_u32_e32 vcc_lo, s6, v1
+; GFX10-NEXT: v_add_co_u32 v8, s4, v4, s6
+; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s4, s7, v5, s4
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s9
; GFX10-NEXT: s_add_i32 s6, s6, 1
-; GFX10-NEXT: v_add_co_u32 v8, s4, v4, v8
-; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s4, v5, v9, s4
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: flat_store_byte v[8:9], v0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
index 01417d7dbe468..e8ed7c20a6b33 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
@@ -90,10 +90,10 @@ define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(ptr addrspace(1) %ptr
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s2, 3
; GFX9-NEXT: s_lshl_b32 s0, s0, 4
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: s_mov_b32 s1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -106,10 +106,10 @@ define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(ptr addrspace(1) %ptr
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s2, 3
; GFX8-NEXT: s_lshl_b32 s0, s0, 4
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: s_mov_b32 s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -136,11 +136,10 @@ define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(ptr addrspace(1) %ptr
; GFX10-LABEL: extractelement_vgpr_v4i128_sgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s2, 3
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: s_mov_b32 s1, 0
; GFX10-NEXT: s_lshl_b32 s0, s0, 4
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -152,12 +151,11 @@ define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(ptr addrspace(1) %ptr
; GFX11-LABEL: extractelement_vgpr_v4i128_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -251,10 +249,9 @@ define amdgpu_ps i128 @extractelement_sgpr_v4i128_vgpr_idx(ptr addrspace(4) inre
; GFX8-LABEL: extractelement_sgpr_v4i128_vgpr_idx:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll
index 89085bec4cf56..6d9d859a6090b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i16.ll
@@ -75,10 +75,10 @@ define amdgpu_ps i16 @extractelement_vgpr_v4i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s2, 3
; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: s_mov_b32 s1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX9-NEXT: global_load_ushort v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -88,10 +88,10 @@ define amdgpu_ps i16 @extractelement_vgpr_v4i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s2, 3
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: s_mov_b32 s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -112,11 +112,10 @@ define amdgpu_ps i16 @extractelement_vgpr_v4i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX10-LABEL: extractelement_vgpr_v4i16_sgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s2, 3
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: s_mov_b32 s1, 0
; GFX10-NEXT: s_lshl_b32 s0, s0, 1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: global_load_ushort v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -125,12 +124,11 @@ define amdgpu_ps i16 @extractelement_vgpr_v4i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX11-LABEL: extractelement_vgpr_v4i16_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX11-NEXT: global_load_u16 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -218,10 +216,9 @@ define amdgpu_ps i16 @extractelement_sgpr_v4i16_vgpr_idx(ptr addrspace(4) inreg
; GFX8-LABEL: extractelement_sgpr_v4i16_vgpr_idx:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -715,10 +712,10 @@ define amdgpu_ps i16 @extractelement_vgpr_v8i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s2, 7
; GFX9-NEXT: s_lshl_b32 s0, s0, 1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: s_mov_b32 s1, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX9-NEXT: global_load_ushort v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -728,10 +725,10 @@ define amdgpu_ps i16 @extractelement_vgpr_v8i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s2, 7
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: s_mov_b32 s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -752,11 +749,10 @@ define amdgpu_ps i16 @extractelement_vgpr_v8i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX10-LABEL: extractelement_vgpr_v8i16_sgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s2, 7
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: s_mov_b32 s1, 0
; GFX10-NEXT: s_lshl_b32 s0, s0, 1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: global_load_ushort v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -765,12 +761,11 @@ define amdgpu_ps i16 @extractelement_vgpr_v8i16_sgpr_idx(ptr addrspace(1) %ptr,
; GFX11-LABEL: extractelement_vgpr_v8i16_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX11-NEXT: global_load_u16 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -858,10 +853,9 @@ define amdgpu_ps i16 @extractelement_sgpr_v8i16_vgpr_idx(ptr addrspace(4) inreg
; GFX8-LABEL: extractelement_sgpr_v8i16_vgpr_idx:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
index faf2898562d57..c60b8be55ccc2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
@@ -79,10 +79,9 @@ define amdgpu_ps i8 @extractelement_vgpr_v4i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s2, 3
; GFX9-NEXT: s_ashr_i32 s1, s0, 31
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -92,10 +91,9 @@ define amdgpu_ps i8 @extractelement_vgpr_v4i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s2, 3
; GFX8-NEXT: s_ashr_i32 s1, s0, 31
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -116,10 +114,8 @@ define amdgpu_ps i8 @extractelement_vgpr_v4i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s2, 3
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -128,12 +124,10 @@ define amdgpu_ps i8 @extractelement_vgpr_v4i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX11-LABEL: extractelement_vgpr_v4i8_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: s_ashr_i32 s1, s0, 31
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -210,12 +204,11 @@ define i8 @extractelement_vgpr_v4i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v2, 3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -223,12 +216,11 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX8-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v2, 3, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc
; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -249,12 +241,10 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX10-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v2, 3, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -262,13 +252,11 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX11-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 3, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -732,10 +720,9 @@ define amdgpu_ps i8 @extractelement_vgpr_v8i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s2, 7
; GFX9-NEXT: s_ashr_i32 s1, s0, 31
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -745,10 +732,9 @@ define amdgpu_ps i8 @extractelement_vgpr_v8i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s2, 7
; GFX8-NEXT: s_ashr_i32 s1, s0, 31
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -769,10 +755,8 @@ define amdgpu_ps i8 @extractelement_vgpr_v8i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s2, 7
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -781,12 +765,10 @@ define amdgpu_ps i8 @extractelement_vgpr_v8i8_sgpr_idx(ptr addrspace(1) %ptr, i3
; GFX11-LABEL: extractelement_vgpr_v8i8_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: s_ashr_i32 s1, s0, 31
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -863,12 +845,11 @@ define i8 @extractelement_vgpr_v8i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v2, 7, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -876,12 +857,11 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX8-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v2, 7, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc
; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -902,12 +882,10 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX10-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v2, 7, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -915,13 +893,11 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX11-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 7, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -1769,10 +1745,9 @@ define amdgpu_ps i8 @extractelement_vgpr_v16i8_sgpr_idx(ptr addrspace(1) %ptr, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s2, 15
; GFX9-NEXT: s_ashr_i32 s1, s0, 31
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -1782,10 +1757,9 @@ define amdgpu_ps i8 @extractelement_vgpr_v16i8_sgpr_idx(ptr addrspace(1) %ptr, i
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s0, s2, 15
; GFX8-NEXT: s_ashr_i32 s1, s0, 31
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -1806,10 +1780,8 @@ define amdgpu_ps i8 @extractelement_vgpr_v16i8_sgpr_idx(ptr addrspace(1) %ptr, i
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s2, 15
; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -1818,12 +1790,10 @@ define amdgpu_ps i8 @extractelement_vgpr_v16i8_sgpr_idx(ptr addrspace(1) %ptr, i
; GFX11-LABEL: extractelement_vgpr_v16i8_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s2, 15
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: s_ashr_i32 s1, s0, 31
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@@ -1900,12 +1870,11 @@ define i8 @extractelement_vgpr_v16i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v2, 15, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -1913,12 +1882,11 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %
;
; GFX8-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v2, 15, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc
; GFX8-NEXT: flat_load_ubyte v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
@@ -1939,12 +1907,10 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %
;
; GFX10-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v2, 15, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -1952,13 +1918,11 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %
;
; GFX11-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 15, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX11-NEXT: global_load_u8 v0, v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
index bff28f14bbfa5..6eb7462ba59ac 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -2474,68 +2474,57 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v_bitcast(<14 x float> inreg %userD
;
; GFX10-LABEL: dyn_extract_v7f64_s_v_bitcast:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mov_b32_e32 v3, s4
-; GFX10-NEXT: v_mov_b32_e32 v4, s5
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s6
-; GFX10-NEXT: v_mov_b32_e32 v6, s7
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s3, v2, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, s8
-; GFX10-NEXT: v_mov_b32_e32 v2, s9
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v5, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v4, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, s10
-; GFX10-NEXT: v_mov_b32_e32 v4, s11
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s9, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, s12
-; GFX10-NEXT: v_mov_b32_e32 v2, s13
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s11, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, s14
-; GFX10-NEXT: v_mov_b32_e32 v4, s15
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s12, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s13, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s14, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s15, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v2, s1, vcc_lo
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: dyn_extract_v7f64_s_v_bitcast:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s5
+; GFX11-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s5
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s7
-; GFX11-NEXT: v_dual_cndmask_b32 v3, v1, v3 :: v_dual_cndmask_b32 v4, v2, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v2, s3, v2, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s9
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v3, v5 :: v_dual_cndmask_b32 v6, v4, v6
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s6, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT: v_dual_mov_b32 v3, s10 :: v_dual_mov_b32 v4, s11
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v1 :: v_dual_cndmask_b32 v6, v6, v2
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s8, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s9, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX11-NEXT: v_dual_mov_b32 v1, s12 :: v_dual_mov_b32 v2, s13
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v3 :: v_dual_cndmask_b32 v6, v6, v4
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s11, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-NEXT: v_dual_mov_b32 v3, s14 :: v_dual_mov_b32 v4, s15
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v2, v6, v2
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s12, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s13, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v2, v2, v4
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s14, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s15, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT: v_dual_cndmask_b32 v0, v1, v0 :: v_dual_cndmask_b32 v1, v2, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v2, s1, vcc_lo
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-NEXT: ; return to shader part epilog
@@ -2628,8 +2617,8 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s14, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s15, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v2, s1, vcc_lo
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-NEXT: ; return to shader part epilog
@@ -2656,7 +2645,8 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s14, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s15, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT: v_dual_cndmask_b32 v0, v1, v0 :: v_dual_cndmask_b32 v1, v2, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v2, s1, vcc_lo
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-NEXT: ; return to shader part epilog
@@ -2714,8 +2704,8 @@ define double @dyn_extract_v7f64_v_v(<7 x double> %vec, i32 %sel) {
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v12, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v13, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v14
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s5, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: dyn_extract_v7f64_v_v:
@@ -2734,7 +2724,8 @@ define double @dyn_extract_v7f64_v_v(<7 x double> %vec, i32 %sel) {
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v14
; GFX11-NEXT: v_dual_cndmask_b32 v0, v0, v12 :: v_dual_cndmask_b32 v1, v1, v13
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v14
-; GFX11-NEXT: v_dual_cndmask_b32 v0, v0, v0 :: v_dual_cndmask_b32 v1, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s1, vcc_lo
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
%ext = extractelement <7 x double> %vec, i32 %sel
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
index 74b3504aef1f5..adf87b383cc73 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
@@ -127,9 +127,8 @@ define amdgpu_ps void @v_fabs_f64(double %in, ptr addrspace(1) %out) {
define amdgpu_ps void @s_fabs_f64(double inreg %in, ptr addrspace(1) %out) {
; GCN-LABEL: s_fabs_f64:
; GCN: ; %bb.0:
-; GCN-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; GCN-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_and_b32_e64 v3, 0x7fffffff, s1
; GCN-NEXT: global_store_b64 v[0:1], v[2:3], off
; GCN-NEXT: s_endpgm
%fabs = call double @llvm.fabs.f64(double %in)
@@ -139,13 +138,10 @@ define amdgpu_ps void @s_fabs_f64(double inreg %in, ptr addrspace(1) %out) {
define amdgpu_ps void @s_fabs_f64_salu_use(double inreg %in, i32 inreg %val, ptr addrspace(1) %out) {
; GFX11-LABEL: s_fabs_f64_salu_use:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_and_b32_e64 v2, 0x7fffffff, s1
; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s1, v3
+; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
@@ -153,13 +149,10 @@ define amdgpu_ps void @s_fabs_f64_salu_use(double inreg %in, i32 inreg %val, ptr
;
; GFX12-LABEL: s_fabs_f64_salu_use:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX12-NEXT: v_and_b32_e64 v2, 0x7fffffff, s1
; GFX12-NEXT: s_cmp_eq_u32 s2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_readfirstlane_b32 s1, v3
+; GFX12-NEXT: v_readfirstlane_b32 s1, v2
; GFX12-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/floor.f64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/floor.f64.ll
index 058ef3fddf69d..b4393c8f44651 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/floor.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/floor.f64.ll
@@ -197,10 +197,10 @@ define amdgpu_ps <2 x float> @s_floor_f64(double inreg %x) {
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_add_f64 v[0:1], s[2:3], -v[0:1]
+; GFX6-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: v_mov_b32_e32 v0, s0
@@ -231,10 +231,10 @@ define amdgpu_ps <2 x float> @s_floor_f64_fneg(double inreg %x) {
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_add_f64 v[0:1], -s[2:3], -v[0:1]
+; GFX6-NEXT: v_add_f64 v[0:1], -v[2:3], -s[0:1]
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: v_mov_b32_e32 v0, s0
@@ -266,10 +266,10 @@ define amdgpu_ps <2 x float> @s_floor_f64_fabs(double inreg %x) {
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_add_f64 v[0:1], |s[2:3]|, -v[0:1]
+; GFX6-NEXT: v_add_f64 v[0:1], |v[2:3]|, -s[0:1]
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: v_mov_b32_e32 v0, s0
@@ -301,10 +301,10 @@ define amdgpu_ps <2 x float> @s_floor_f64_fneg_fabs(double inreg %x) {
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
; GFX6-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_add_f64 v[0:1], -|s[2:3]|, -v[0:1]
+; GFX6-NEXT: v_add_f64 v[0:1], -|v[2:3]|, -s[0:1]
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-NEXT: v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
index 4dff8cfcec830..ce2d24577576d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
@@ -127,9 +127,8 @@ define amdgpu_ps void @v_fneg_f64(double %in, ptr addrspace(1) %out) {
define amdgpu_ps void @s_fneg_f64(double inreg %in, ptr addrspace(1) %out) {
; GCN-LABEL: s_fneg_f64:
; GCN: ; %bb.0:
-; GCN-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GCN-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_xor_b32_e64 v3, 0x80000000, s1
; GCN-NEXT: global_store_b64 v[0:1], v[2:3], off
; GCN-NEXT: s_endpgm
%fneg = fneg double %in
@@ -139,13 +138,10 @@ define amdgpu_ps void @s_fneg_f64(double inreg %in, ptr addrspace(1) %out) {
define amdgpu_ps void @s_fneg_f64_salu_use(double inreg %in, i32 inreg %val, ptr addrspace(1) %out) {
; GFX11-LABEL: s_fneg_f64_salu_use:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_xor_b32_e64 v2, 0x80000000, s1
; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s1, v3
+; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
@@ -153,13 +149,10 @@ define amdgpu_ps void @s_fneg_f64_salu_use(double inreg %in, i32 inreg %val, ptr
;
; GFX12-LABEL: s_fneg_f64_salu_use:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX12-NEXT: v_xor_b32_e64 v2, 0x80000000, s1
; GFX12-NEXT: s_cmp_eq_u32 s2, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_readfirstlane_b32 s1, v3
+; GFX12-NEXT: v_readfirstlane_b32 s1, v2
; GFX12-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
index b84ec70d3e35d..ad6d0cac5e284 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
@@ -1378,16 +1378,16 @@ define float @v_pow_f32_fabs_lhs_rhs(float %x, float %y) {
define amdgpu_ps float @v_pow_f32_sgpr_vgpr(float inreg %x, float %y) {
; GFX6-LABEL: v_pow_f32_sgpr_vgpr:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x800000
-; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
-; GFX6-NEXT: s_or_b64 s[2:3], vcc, vcc
-; GFX6-NEXT: s_cselect_b32 s2, 1, 0
+; GFX6-NEXT: v_mov_b32_e32 v2, 0x800000
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, s0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: s_or_b64 s[0:1], vcc, vcc
; GFX6-NEXT: s_cselect_b32 s1, 1, 0
-; GFX6-NEXT: s_lshl_b32 s2, s2, 5
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_ldexp_f32_e32 v1, s0, v1
+; GFX6-NEXT: s_cselect_b32 s0, 1, 0
+; GFX6-NEXT: s_lshl_b32 s1, s1, 5
+; GFX6-NEXT: v_ldexp_f32_e64 v1, v1, s1
; GFX6-NEXT: v_log_f32_e32 v1, v1
-; GFX6-NEXT: s_cmp_lg_u32 s1, 0
+; GFX6-NEXT: s_cmp_lg_u32 s0, 0
; GFX6-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX6-NEXT: v_mov_b32_e32 v2, 0x42800000
; GFX6-NEXT: v_subrev_f32_e32 v1, s0, v1
@@ -1404,16 +1404,16 @@ define amdgpu_ps float @v_pow_f32_sgpr_vgpr(float inreg %x, float %y) {
;
; GFX8-LABEL: v_pow_f32_sgpr_vgpr:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x800000
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 0x800000
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, s0, v2
; GFX8-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX8-NEXT: s_cselect_b32 s2, 1, 0
; GFX8-NEXT: s_cselect_b32 s1, 1, 0
-; GFX8-NEXT: s_lshl_b32 s2, s2, 5
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_ldexp_f32 v1, s0, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 5
+; GFX8-NEXT: v_ldexp_f32 v1, v1, s1
; GFX8-NEXT: v_log_f32_e32 v1, v1
-; GFX8-NEXT: s_cmp_lg_u32 s1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s0, 0
; GFX8-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX8-NEXT: v_mov_b32_e32 v2, 0x42800000
; GFX8-NEXT: v_subrev_f32_e32 v1, s0, v1
@@ -1430,16 +1430,16 @@ define amdgpu_ps float @v_pow_f32_sgpr_vgpr(float inreg %x, float %y) {
;
; GFX9-LABEL: v_pow_f32_sgpr_vgpr:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x800000
-; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x800000
+; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, s0, v2
; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT: s_cselect_b32 s2, 1, 0
; GFX9-NEXT: s_cselect_b32 s1, 1, 0
-; GFX9-NEXT: s_lshl_b32 s2, s2, 5
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_ldexp_f32 v1, s0, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 5
+; GFX9-NEXT: v_ldexp_f32 v1, v1, s1
; GFX9-NEXT: v_log_f32_e32 v1, v1
-; GFX9-NEXT: s_cmp_lg_u32 s1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s0, 0
; GFX9-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0x42800000
; GFX9-NEXT: v_subrev_f32_e32 v1, s0, v1
@@ -1622,16 +1622,16 @@ define amdgpu_ps float @v_pow_f32_vgpr_sgpr(float %x, float inreg %y) {
define amdgpu_ps float @v_pow_f32_sgpr_sgpr(float inreg %x, float inreg %y) {
; GFX6-LABEL: v_pow_f32_sgpr_sgpr:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GFX6-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX6-NEXT: s_or_b64 s[2:3], vcc, vcc
-; GFX6-NEXT: s_cselect_b32 s3, 1, 0
; GFX6-NEXT: s_cselect_b32 s2, 1, 0
-; GFX6-NEXT: s_lshl_b32 s3, s3, 5
-; GFX6-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-NEXT: v_ldexp_f32_e32 v0, s0, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: s_cselect_b32 s0, 1, 0
+; GFX6-NEXT: s_lshl_b32 s2, s2, 5
+; GFX6-NEXT: v_ldexp_f32_e64 v0, v0, s2
; GFX6-NEXT: v_log_f32_e32 v0, v0
-; GFX6-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-NEXT: s_cmp_lg_u32 s0, 0
; GFX6-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX6-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
; GFX6-NEXT: v_subrev_f32_e32 v0, s0, v0
@@ -1647,16 +1647,16 @@ define amdgpu_ps float @v_pow_f32_sgpr_sgpr(float inreg %x, float inreg %y) {
;
; GFX8-LABEL: v_pow_f32_sgpr_sgpr:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX8-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX8-NEXT: s_cselect_b32 s3, 1, 0
; GFX8-NEXT: s_cselect_b32 s2, 1, 0
-; GFX8-NEXT: s_lshl_b32 s3, s3, 5
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_ldexp_f32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 5
+; GFX8-NEXT: v_ldexp_f32 v0, v0, s2
; GFX8-NEXT: v_log_f32_e32 v0, v0
-; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_cmp_lg_u32 s0, 0
; GFX8-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
; GFX8-NEXT: v_subrev_f32_e32 v0, s0, v0
@@ -1672,16 +1672,16 @@ define amdgpu_ps float @v_pow_f32_sgpr_sgpr(float inreg %x, float inreg %y) {
;
; GFX9-LABEL: v_pow_f32_sgpr_sgpr:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX9-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT: s_cselect_b32 s3, 1, 0
; GFX9-NEXT: s_cselect_b32 s2, 1, 0
-; GFX9-NEXT: s_lshl_b32 s3, s3, 5
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_ldexp_f32 v0, s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: s_lshl_b32 s2, s2, 5
+; GFX9-NEXT: v_ldexp_f32 v0, v0, s2
; GFX9-NEXT: v_log_f32_e32 v0, v0
-; GFX9-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-NEXT: s_cmp_lg_u32 s0, 0
; GFX9-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
; GFX9-NEXT: v_subrev_f32_e32 v0, s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index 84a7ad4a65ac7..e582b0e8d098b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -489,9 +489,7 @@ define amdgpu_ps <2 x half> @fptrunc_v2f32_to_v2f16_uniform(<2 x float> inreg %a
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_f16_f32 v0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
%result = fptrunc <2 x float> %a to <2 x half>
ret <2 x half> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 6844b698473c7..26fadf5e36f97 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -5949,37 +5949,31 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX6-LABEL: v_fshl_i64_svs:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], 1
+; GFX6-NEXT: s_andn2_b32 s3, 63, s2
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: s_andn2_b32 s2, 63, s2
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i64_svs:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX8-NEXT: s_andn2_b32 s3, 63, s2
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: s_andn2_b32 s2, 63, s2
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i64_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX9-NEXT: s_andn2_b32 s3, 63, s2
+; GFX9-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: s_andn2_b32 s2, 63, s2
-; GFX9-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i64_svs:
@@ -5987,11 +5981,9 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
; GFX10-NEXT: s_andn2_b32 s3, 63, s2
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i64_svs:
@@ -5999,12 +5991,11 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -6015,66 +6006,59 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX6-LABEL: v_fshl_i64_vss:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_and_b32 s3, s2, 63
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
; GFX6-NEXT: s_not_b32 s2, s2
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i64_vss:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_and_b32 s3, s2, 63
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
; GFX8-NEXT: s_not_b32 s2, s2
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i64_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 63
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
; GFX9-NEXT: s_not_b32 s2, s2
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i64_vss:
; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s3, s2, 63
; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_not_b32 s3, s2
-; GFX10-NEXT: s_and_b32 s2, s2, 63
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX10-NEXT: s_not_b32 s2, s2
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i64_vss:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s2, 63
; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s3, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 63
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-NEXT: s_not_b32 s2, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -6911,42 +6895,40 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffc0, v11
; GFX10-NEXT: v_lshlrev_b64 v[1:2], v11, s[2:3]
; GFX10-NEXT: v_sub_nc_u32_e32 v9, 64, v12
-; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX10-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GFX10-NEXT: v_lshrrev_b64 v[3:4], v3, s[0:1]
; GFX10-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
; GFX10-NEXT: v_lshlrev_b64 v[7:8], v7, s[0:1]
-; GFX10-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v11
; GFX10-NEXT: v_add_nc_u32_e32 v13, 0xffffffc0, v12
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v1
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], v12, s[4:5]
; GFX10-NEXT: v_lshlrev_b64 v[9:10], v9, s[6:7]
+; GFX10-NEXT: v_or_b32_e32 v3, v3, v1
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v12, s[8:9]
; GFX10-NEXT: v_or_b32_e32 v4, v4, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v14, 0, v5, vcc_lo
+; GFX10-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
+; GFX10-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v3, vcc_lo
; GFX10-NEXT: v_lshrrev_b64 v[2:3], v13, s[6:7]
-; GFX10-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
-; GFX10-NEXT: v_or_b32_e32 v5, v0, v9
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v9
; GFX10-NEXT: v_or_b32_e32 v9, v1, v10
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v4, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v11
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v5, s1
-; GFX10-NEXT: v_lshrrev_b64 v[4:5], v12, s[6:7]
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v12
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v0, s1
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], v12, s[6:7]
; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v9, s1
+; GFX10-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, s3, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v0, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, v1, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, v4, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, v5, s1
-; GFX10-NEXT: v_or_b32_e32 v0, v14, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v6, v1
-; GFX10-NEXT: v_or_b32_e32 v2, v7, v2
-; GFX10-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, s3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s8, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s9, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, v0, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, v1, s1
+; GFX10-NEXT: v_or_b32_e32 v0, v5, v2
+; GFX10-NEXT: v_or_b32_e32 v1, v6, v3
+; GFX10-NEXT: v_or_b32_e32 v2, v7, v8
+; GFX10-NEXT: v_or_b32_e32 v3, v4, v9
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i128_ssv:
@@ -6959,48 +6941,47 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX11-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v11
; GFX11-NEXT: v_sub_nc_u32_e32 v9, 64, v12
-; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX11-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_cndmask_b32 v14, 0, v5 :: v_dual_add_nc_u32 v13, 0xffffffc0, v12
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v6 :: v_dual_add_nc_u32 v13, 0xffffffc0, v12
; GFX11-NEXT: v_sub_nc_u32_e32 v3, 64, v11
; GFX11-NEXT: v_lshlrev_b64 v[1:2], v11, s[2:3]
; GFX11-NEXT: v_lshlrev_b64 v[9:10], v9, s[6:7]
-; GFX11-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 0, v12
+; GFX11-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
; GFX11-NEXT: v_lshrrev_b64 v[3:4], v3, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v4, v4, v2
; GFX11-NEXT: v_add_nc_u32_e32 v7, 0xffffffc0, v11
; GFX11-NEXT: v_or_b32_e32 v3, v3, v1
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v12, s[4:5]
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v12, s[8:9]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_lshlrev_b64 v[7:8], v7, s[0:1]
; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v11
-; GFX11-NEXT: v_or_b32_e32 v5, v0, v9
-; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v9
; GFX11-NEXT: v_or_b32_e32 v9, v1, v10
; GFX11-NEXT: v_cndmask_b32_e32 v7, v7, v3, vcc_lo
; GFX11-NEXT: v_lshrrev_b64 v[2:3], v13, s[6:7]
-; GFX11-NEXT: v_dual_cndmask_b32 v8, v8, v4 :: v_dual_mov_b32 v1, s5
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 0, v12
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v5, s1
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v12, s[6:7]
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v0, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v12, s[6:7]
; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v9, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, s3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, s3, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v0, s4
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, v1, s4
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v4, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v3, 0, v5, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s8, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s9, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v14, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v5, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v6, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v7, v2
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX11-NEXT: v_or_b32_e32 v2, v7, v8
+; GFX11-NEXT: v_or_b32_e32 v3, v4, v9
; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
@@ -7051,19 +7032,15 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-NEXT: s_cmp_lg_u32 s5, 0
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX6-NEXT: v_mov_b32_e32 v0, s6
-; GFX6-NEXT: v_mov_b32_e32 v1, s7
-; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_mov_b32_e32 v3, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v1, s7, v1
+; GFX6-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX6-NEXT: v_or_b32_e32 v3, s1, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i128_svs:
@@ -7109,19 +7086,15 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
; GFX8-NEXT: s_cmp_lg_u32 s5, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, s6, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s7, v1
+; GFX8-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s1, v3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i128_svs:
@@ -7166,19 +7139,15 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
; GFX9-NEXT: s_cmp_lg_u32 s5, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, s6, v0
+; GFX9-NEXT: v_or_b32_e32 v1, s7, v1
+; GFX9-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX9-NEXT: v_or_b32_e32 v3, s1, v3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i128_svs:
@@ -7223,19 +7192,15 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-NEXT: v_mov_b32_e32 v1, s9
-; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v0, s8, v0
+; GFX10-NEXT: v_or_b32_e32 v1, s9, v1
+; GFX10-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX10-NEXT: v_or_b32_e32 v3, s1, v3
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i128_svs:
@@ -7280,18 +7245,16 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_dual_cndmask_b32 v2, 0, v8 :: v_dual_cndmask_b32 v3, 0, v9
+; GFX11-NEXT: v_or_b32_e32 v0, s8, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, s9, v1
+; GFX11-NEXT: v_or_b32_e32 v2, s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: v_or_b32_e32 v3, s1, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
@@ -7346,17 +7309,13 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX6-NEXT: s_cmp_lg_u32 s12, 0
; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GFX6-NEXT: s_cmp_lg_u32 s11, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
; GFX6-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s3
-; GFX6-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v6
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX6-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX6-NEXT: v_or_b32_e32 v3, s3, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i128_vss:
@@ -7406,17 +7365,13 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX8-NEXT: s_cmp_lg_u32 s12, 0
; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GFX8-NEXT: s_cmp_lg_u32 s11, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
; GFX8-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v6
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX8-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s3, v3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshl_i128_vss:
@@ -7466,137 +7421,124 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX9-NEXT: s_cmp_lg_u32 s12, 0
; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
; GFX9-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX9-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX9-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v6
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX9-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX9-NEXT: v_or_b32_e32 v3, s3, v3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshl_i128_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s5, s4, 0x7f
-; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: s_sub_i32 s6, 64, s5
; GFX10-NEXT: s_sub_i32 s7, s5, 64
; GFX10-NEXT: s_cmp_lt_u32 s5, 64
; GFX10-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
-; GFX10-NEXT: s_cselect_b32 s6, 1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s5, 0
; GFX10-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
-; GFX10-NEXT: s_cselect_b32 s9, 1, 0
-; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: s_cselect_b32 s8, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s5, 0
; GFX10-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
-; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
-; GFX10-NEXT: s_cmp_lg_u32 s9, 0
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
-; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_lshl_b32 s9, s2, 31
-; GFX10-NEXT: s_andn2_b32 s7, 0x7f, s4
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX10-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: s_not_b32 s10, s4
-; GFX10-NEXT: s_sub_i32 s12, s7, 64
-; GFX10-NEXT: s_sub_i32 s8, 64, s7
-; GFX10-NEXT: s_cmp_lt_u32 s7, 64
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
-; GFX10-NEXT: s_cselect_b32 s13, 1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s7, 0
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
-; GFX10-NEXT: s_cselect_b32 s7, 1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[0:1], s10
-; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
-; GFX10-NEXT: s_lshr_b64 s[10:11], s[2:3], s10
-; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
-; GFX10-NEXT: s_cmp_lg_u32 s13, 0
; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: s_lshl_b32 s7, s2, 31
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT: s_andn2_b32 s6, 0x7f, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT: s_not_b32 s8, s4
+; GFX10-NEXT: s_sub_i32 s10, s6, 64
+; GFX10-NEXT: s_sub_i32 s7, 64, s6
+; GFX10-NEXT: s_cmp_lt_u32 s6, 64
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, v2, s5
+; GFX10-NEXT: s_cselect_b32 s11, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s6, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v1, v3, s5
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[0:1], s8
+; GFX10-NEXT: s_lshl_b64 s[6:7], s[2:3], s7
+; GFX10-NEXT: s_lshr_b64 s[8:9], s[2:3], s8
+; GFX10-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX10-NEXT: s_cmp_lg_u32 s11, 0
; GFX10-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
-; GFX10-NEXT: s_cmp_lg_u32 s7, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-NEXT: s_cmp_lg_u32 s13, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_cselect_b64 s[2:3], s[10:11], 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v5, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
-; GFX10-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX10-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v6
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[8:9], 0
+; GFX10-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX10-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX10-NEXT: v_or_b32_e32 v3, s3, v3
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_i128_vss:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
-; GFX11-NEXT: s_mov_b32 s8, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_sub_i32 s6, 64, s5
; GFX11-NEXT: s_sub_i32 s7, s5, 64
; GFX11-NEXT: s_cmp_lt_u32 s5, 64
; GFX11-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
-; GFX11-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s5, 0
; GFX11-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
-; GFX11-NEXT: s_cselect_b32 s9, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
; GFX11-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX11-NEXT: s_cselect_b32 s9, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 0
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: s_cselect_b32 s5, exec_lo, 0
-; GFX11-NEXT: s_cmp_lg_u32 s9, 0
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
; GFX11-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
-; GFX11-NEXT: s_cselect_b32 s6, exec_lo, 0
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s9, 0
+; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_cselect_b32 s5, exec_lo, 0
; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_lshl_b32 s9, s2, 31
-; GFX11-NEXT: s_and_not1_b32 s7, 0x7f, s4
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX11-NEXT: s_lshl_b32 s7, s2, 31
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v0, v4 :: v_dual_cndmask_b32 v1, v1, v5
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX11-NEXT: s_and_not1_b32 s6, 0x7f, s4
; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_not_b32 s10, s4
-; GFX11-NEXT: s_sub_i32 s12, s7, 64
-; GFX11-NEXT: s_sub_i32 s8, 64, s7
-; GFX11-NEXT: s_cmp_lt_u32 s7, 64
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
-; GFX11-NEXT: s_cselect_b32 s13, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s7, 0
-; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_cndmask_b32 v7, 0, v9
-; GFX11-NEXT: s_cselect_b32 s7, 1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], s10
-; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[2:3], s10
-; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
-; GFX11-NEXT: s_cmp_lg_u32 s13, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX11-NEXT: s_not_b32 s8, s4
+; GFX11-NEXT: s_sub_i32 s10, s6, 64
+; GFX11-NEXT: s_sub_i32 s7, 64, s6
+; GFX11-NEXT: s_cmp_lt_u32 s6, 64
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, v2, s5
+; GFX11-NEXT: s_cselect_b32 s11, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s6, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v1, v3, s5
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], s8
+; GFX11-NEXT: s_lshl_b64 s[6:7], s[2:3], s7
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], s8
+; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX11-NEXT: s_cmp_lg_u32 s11, 0
; GFX11-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
-; GFX11-NEXT: s_cmp_lg_u32 s7, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
+; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
-; GFX11-NEXT: s_cmp_lg_u32 s13, 0
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: s_cselect_b64 s[2:3], s[10:11], 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
-; GFX11-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX11-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v6
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[8:9], 0
+; GFX11-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX11-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX11-NEXT: v_or_b32_e32 v3, s3, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 11b62d825c965..b2200a00da1bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -5657,67 +5657,60 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg %amt) {
; GFX6-LABEL: v_fshr_i64_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
; GFX6-NEXT: s_not_b32 s3, s2
; GFX6-NEXT: s_and_b32 s2, s2, 63
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i64_svs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
; GFX8-NEXT: s_not_b32 s3, s2
; GFX8-NEXT: s_and_b32 s2, s2, 63
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
; GFX8-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i64_svs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
; GFX9-NEXT: s_not_b32 s3, s2
; GFX9-NEXT: s_and_b32 s2, s2, 63
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
; GFX9-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i64_svs:
; GFX10: ; %bb.0:
+; GFX10-NEXT: s_and_b32 s3, s2, 63
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT: s_not_b32 s3, s2
-; GFX10-NEXT: s_and_b32 s2, s2, 63
-; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
-; GFX10-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
+; GFX10-NEXT: s_not_b32 s2, s2
+; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i64_svs:
; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s2, 63
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s3, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 63
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
+; GFX11-NEXT: s_not_b32 s2, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -5731,10 +5724,8 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX6-NEXT: s_andn2_b32 s3, 63, s2
; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s3
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i64_vss:
@@ -5743,10 +5734,8 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX8-NEXT: s_andn2_b32 s3, 63, s2
; GFX8-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i64_vss:
@@ -5755,10 +5744,8 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX9-NEXT: s_andn2_b32 s3, 63, s2
; GFX9-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i64_vss:
@@ -5766,11 +5753,9 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
; GFX10-NEXT: s_andn2_b32 s3, 63, s2
; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT: v_or_b32_e32 v1, s1, v1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i64_vss:
@@ -5778,12 +5763,11 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, s1, v1
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
@@ -6755,19 +6739,15 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-NEXT: s_cmp_lg_u32 s7, 0
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s3
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX6-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX6-NEXT: v_or_b32_e32 v3, s3, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i128_svs:
@@ -6814,19 +6794,15 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
; GFX8-NEXT: s_cmp_lg_u32 s7, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX8-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s3, v3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i128_svs:
@@ -6873,19 +6849,15 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
; GFX9-NEXT: s_cmp_lg_u32 s7, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX9-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX9-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX9-NEXT: v_or_b32_e32 v3, s3, v3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i128_svs:
@@ -6932,19 +6904,15 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v2, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v3, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s8
-; GFX10-NEXT: v_mov_b32_e32 v1, s9
-; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v0, s8, v0
+; GFX10-NEXT: v_or_b32_e32 v1, s9, v1
+; GFX10-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX10-NEXT: v_or_b32_e32 v3, s1, v3
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i128_svs:
@@ -6991,18 +6959,16 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_dual_cndmask_b32 v2, 0, v8 :: v_dual_cndmask_b32 v3, 0, v9
+; GFX11-NEXT: v_or_b32_e32 v0, s8, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, s9, v1
+; GFX11-NEXT: v_or_b32_e32 v2, s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: v_or_b32_e32 v3, s1, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
@@ -7055,17 +7021,13 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX6-NEXT: s_cmp_lg_u32 s12, 0
; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GFX6-NEXT: s_cmp_lg_u32 s11, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
; GFX6-NEXT: s_cselect_b64 s[2:3], s[6:7], 0
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
-; GFX6-NEXT: v_mov_b32_e32 v3, s3
-; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX6-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v4
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v5
+; GFX6-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX6-NEXT: v_or_b32_e32 v3, s3, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i128_vss:
@@ -7113,17 +7075,13 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX8-NEXT: s_cmp_lg_u32 s12, 0
; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GFX8-NEXT: s_cmp_lg_u32 s11, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
; GFX8-NEXT: s_cselect_b64 s[2:3], s[6:7], 0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v4
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v5
+; GFX8-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s3, v3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: v_fshr_i128_vss:
@@ -7171,17 +7129,13 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX9-NEXT: s_cmp_lg_u32 s12, 0
; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
; GFX9-NEXT: s_cselect_b64 s[2:3], s[6:7], 0
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX9-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX9-NEXT: v_or_b32_e32 v3, v7, v3
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v4
+; GFX9-NEXT: v_or_b32_e32 v1, s1, v5
+; GFX9-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX9-NEXT: v_or_b32_e32 v3, s3, v3
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: v_fshr_i128_vss:
@@ -7191,55 +7145,51 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
; GFX10-NEXT: s_andn2_b32 s5, 0x7f, s4
; GFX10-NEXT: s_sub_i32 s6, 64, s5
+; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
; GFX10-NEXT: s_sub_i32 s7, s5, 64
; GFX10-NEXT: s_cmp_lt_u32 s5, 64
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
; GFX10-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
; GFX10-NEXT: s_cselect_b32 s6, 1, 0
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
; GFX10-NEXT: s_cmp_eq_u32 s5, 0
; GFX10-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
; GFX10-NEXT: s_cselect_b32 s8, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s6, 0
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
-; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
-; GFX10-NEXT: s_cmp_lg_u32 s8, 0
; GFX10-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
-; GFX10-NEXT: s_cselect_b32 s6, exec_lo, 0
-; GFX10-NEXT: s_and_b32 s7, s4, 0x7f
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX10-NEXT: s_sub_i32 s12, s7, 64
-; GFX10-NEXT: s_sub_i32 s10, 64, s7
-; GFX10-NEXT: s_cmp_lt_u32 s7, 64
; GFX10-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX10-NEXT: s_cselect_b32 s13, 1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s7, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
-; GFX10-NEXT: s_cselect_b32 s7, 1, 0
-; GFX10-NEXT: s_lshr_b64 s[8:9], s[0:1], s4
-; GFX10-NEXT: s_lshl_b64 s[10:11], s[2:3], s10
; GFX10-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
-; GFX10-NEXT: s_lshr_b64 s[4:5], s[2:3], s4
-; GFX10-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
-; GFX10-NEXT: s_cmp_lg_u32 s13, 0
; GFX10-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc_lo
-; GFX10-NEXT: s_cselect_b64 s[2:3], s[8:9], s[2:3]
-; GFX10-NEXT: s_cmp_lg_u32 s7, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX10-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX10-NEXT: s_cmp_lg_u32 s8, 0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX10-NEXT: s_and_b32 s6, s4, 0x7f
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX10-NEXT: s_sub_i32 s10, s6, 64
+; GFX10-NEXT: s_sub_i32 s8, 64, s6
+; GFX10-NEXT: s_cmp_lt_u32 s6, 64
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, v2, s5
+; GFX10-NEXT: s_cselect_b32 s11, 1, 0
+; GFX10-NEXT: s_cmp_eq_u32 s6, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v1, v3, s5
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_lshr_b64 s[6:7], s[0:1], s4
+; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[2:3], s4
+; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX10-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-NEXT: s_cmp_lg_u32 s13, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10-NEXT: v_or_b32_e32 v0, s0, v6
; GFX10-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v5, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
-; GFX10-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX10-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX10-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX10-NEXT: v_or_b32_e32 v3, s3, v3
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_i128_vss:
@@ -7248,57 +7198,52 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX11-NEXT: v_lshrrev_b32_e32 v4, 31, v1
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
; GFX11-NEXT: s_and_not1_b32 s5, 0x7f, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
; GFX11-NEXT: s_sub_i32 s7, s5, 64
; GFX11-NEXT: s_cmp_lt_u32 s5, 64
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
; GFX11-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
; GFX11-NEXT: s_cmp_eq_u32 s5, 0
; GFX11-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
; GFX11-NEXT: s_cselect_b32 s8, 1, 0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: s_cselect_b32 s5, exec_lo, 0
-; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
-; GFX11-NEXT: s_cselect_b32 s6, exec_lo, 0
-; GFX11-NEXT: s_and_b32 s7, s4, 0x7f
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX11-NEXT: s_sub_i32 s12, s7, 64
-; GFX11-NEXT: s_sub_i32 s10, 64, s7
-; GFX11-NEXT: s_cmp_lt_u32 s7, 64
; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-NEXT: s_cselect_b32 s13, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s7, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
-; GFX11-NEXT: s_cselect_b32 s7, 1, 0
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[0:1], s4
-; GFX11-NEXT: s_lshl_b64 s[10:11], s[2:3], s10
; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_cndmask_b32 v7, 0, v9
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v0, v4 :: v_dual_cndmask_b32 v1, v1, v5
+; GFX11-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX11-NEXT: s_and_b32 s6, s4, 0x7f
+; GFX11-NEXT: s_sub_i32 s10, s6, 64
+; GFX11-NEXT: s_sub_i32 s8, 64, s6
+; GFX11-NEXT: s_cmp_lt_u32 s6, 64
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, v2, s5
+; GFX11-NEXT: s_cselect_b32 s11, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s6, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v1, v3, s5
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s4
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], s4
-; GFX11-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
-; GFX11-NEXT: s_cmp_lg_u32 s13, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
-; GFX11-NEXT: s_cselect_b64 s[2:3], s[8:9], s[2:3]
-; GFX11-NEXT: s_cmp_lg_u32 s7, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX11-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
-; GFX11-NEXT: s_cmp_lg_u32 s13, 0
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v6
; GFX11-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
-; GFX11-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v7, v1
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX11-NEXT: v_or_b32_e32 v1, s1, v7
+; GFX11-NEXT: v_or_b32_e32 v2, s2, v2
+; GFX11-NEXT: v_or_b32_e32 v3, s3, v3
; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index 0f963ba85b73e..5374f357a4ede 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -1079,51 +1079,49 @@ define amdgpu_ps void @insertelement_s_v4i16_s_v(ptr addrspace(4) inreg %ptr, i1
;
; GFX10-LABEL: insertelement_s_v4i16_s_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX10-NEXT: v_and_b32_e32 v1, 1, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 1, v0
-; GFX10-NEXT: s_and_b32 s2, s4, 0xffff
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 4, v1
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v2, 0xffff
-; GFX10-NEXT: v_lshlrev_b32_e64 v2, v2, s2
-; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX10-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v1, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX10-NEXT: v_not_b32_e32 v2, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX10-NEXT: v_and_or_b32 v5, v5, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: v_mov_b32_e32 v0, s3
+; GFX10-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v4, v0, v2, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s2, v4, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v4, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i16_s_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v1, 1, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 1, v0
-; GFX11-NEXT: s_and_b32 s2, s4, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX11-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 4, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v2, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e64 v2, v2, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_lshlrev_b32 v1, 4, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v1, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX11-NEXT: v_not_b32_e32 v2, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_or_b32 v5, v5, v3, v2
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: v_and_or_b32 v4, v0, v2, v1
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s2, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s3, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <4 x i16> %vec, i16 %val, i32 %idx
@@ -1207,50 +1205,48 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
;
; GFX10-LABEL: insertelement_s_v4i16_v_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX10-NEXT: v_and_b32_e32 v2, 1, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 1, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 4, v2
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
-; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_not_b32_e32 v3, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 1, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v2, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v2, v3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc_lo
-; GFX10-NEXT: v_and_or_b32 v6, v4, v3, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v6, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
-; GFX10-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v3, v1, v2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s2, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v3, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i16_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v2, 1, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 4, v2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, v2, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v3, v4
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
-; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_lshlrev_b32 v2, 4, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v2, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_not_b32_e32 v2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v3, v1, v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s2, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s3, v3, vcc_lo
+; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <4 x i16> %vec, i16 %val, i32 %idx
@@ -2066,65 +2062,61 @@ define amdgpu_ps void @insertelement_s_v8i16_s_v(ptr addrspace(4) inreg %ptr, i1
;
; GFX10-LABEL: insertelement_s_v8i16_s_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 1, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 1, v0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 4, v4
-; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v7, v7
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX10-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v1, 1, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 1, v0
; GFX10-NEXT: s_and_b32 s1, s4, 0xffff
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_e64 v4, v4, s1
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
-; GFX10-NEXT: v_and_or_b32 v7, v5, v7, v4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v1, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v1, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s9
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s8, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s10, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s11, s1
+; GFX10-NEXT: v_and_or_b32 v3, v2, v3, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v0, s8, v3, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s9, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s10, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, s11, v3, s1
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i16_s_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 1, v0
-; GFX11-NEXT: v_and_b32_e32 v4, 1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 4, v4
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX11-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v1, 1, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
; GFX11-NEXT: s_and_b32 s1, s4, 0xffff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
-; GFX11-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v4, s1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_not_b32_e32 v7, v7
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_or_b32 v7, v5, v7, v4
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v2, s9 :: v_dual_lshlrev_b32 v1, 4, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v2, s8, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v1, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v1, s1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s10, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s11, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v3, v2, v3, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, s8, v3, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s9, v3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s10, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, s11, v3, s1
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(4) %ptr
@@ -2233,66 +2225,63 @@ define amdgpu_ps void @insertelement_s_v8i16_v_v(ptr addrspace(4) inreg %ptr, i1
;
; GFX10-LABEL: insertelement_s_v8i16_v_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 1, v1
-; GFX10-NEXT: v_and_b32_e32 v5, 1, v1
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 4, v5
-; GFX10-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_not_b32_e32 v5, v8
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
-; GFX10-NEXT: v_and_or_b32 v8, v6, v5, v0
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v2, 1, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 1, v1
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_mov_b32_e32 v6, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
-; GFX10-NEXT: global_store_dwordx4 v[5:6], v[0:3], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v2, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s4, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s6, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s7, s1
+; GFX10-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, s4, v3, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s5, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s6, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, s7, v3, s1
+; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i16_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v5, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
+; GFX11-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v2, 1, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v4, s3
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 4, v5
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
-; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, v5, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_not_b32_e32 v5, v8
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_or_b32 v8, v6, v5, v0
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
-; GFX11-NEXT: global_store_b128 v[5:6], v[0:3], off
+; GFX11-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_lshlrev_b32 v2, 4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s4, v3, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s6, s0
+; GFX11-NEXT: v_not_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s7, s1
+; GFX11-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, s4, v3, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s5, v3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s6, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, s7, v3, s1
+; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i16> %vec, i16 %val, i32 %idx
@@ -3313,105 +3302,95 @@ define amdgpu_ps void @insertelement_s_v16i16_s_v(ptr addrspace(4) inreg %ptr, i
; GFX10-LABEL: insertelement_s_v16i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX10-NEXT: v_lshrrev_b32_e32 v13, 1, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 1, v0
; GFX10-NEXT: v_and_b32_e32 v0, 1, v0
; GFX10-NEXT: s_and_b32 s5, s4, 0xffff
-; GFX10-NEXT: v_mov_b32_e32 v11, 16
-; GFX10-NEXT: v_mov_b32_e32 v12, 0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v13
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v13
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v13
-; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v13
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: v_mov_b32_e32 v9, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v13
-; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v13
-; GFX10-NEXT: v_lshlrev_b32_e64 v10, v0, 0xffff
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v1
+; GFX10-NEXT: v_mov_b32_e32 v10, 16
+; GFX10-NEXT: v_mov_b32_e32 v11, 0
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX10-NEXT: v_lshlrev_b32_e64 v0, v0, s5
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v13
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, s8
; GFX10-NEXT: v_mov_b32_e32 v2, s9
-; GFX10-NEXT: v_mov_b32_e32 v3, s10
-; GFX10-NEXT: v_mov_b32_e32 v4, s11
-; GFX10-NEXT: v_mov_b32_e32 v5, s12
-; GFX10-NEXT: v_mov_b32_e32 v6, s13
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v1, v2, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v7, s14
-; GFX10-NEXT: v_mov_b32_e32 v8, s15
-; GFX10-NEXT: v_not_b32_e32 v10, v10
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v3, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v4, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v5, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v6, s3
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v8, s5
-; GFX10-NEXT: v_and_or_b32 v14, v9, v10, v0
-; GFX10-NEXT: v_mov_b32_e32 v9, 0
-; GFX10-NEXT: v_mov_b32_e32 v10, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v14, s6
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v14, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, v14, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, v14, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v5, v14, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v6, v14, s3
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v7, v14, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v8, v14, s5
-; GFX10-NEXT: global_store_dwordx4 v[9:10], v[0:3], off
-; GFX10-NEXT: global_store_dwordx4 v[11:12], v[4:7], off
+; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s8, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s10, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s11, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s12, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s13, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s14, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s15, s5
+; GFX10-NEXT: v_and_or_b32 v7, v2, v3, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, s8, v7, s6
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s9, v7, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s10, v7, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, s11, v7, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v4, s12, v7, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v5, s13, v7, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v6, s14, v7, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, s15, v7, s5
+; GFX10-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[8:15], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 1, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v0
; GFX11-NEXT: s_and_b32 s5, s4, 0xffff
-; GFX11-NEXT: v_dual_mov_b32 v11, 16 :: v_dual_and_b32 v0, 1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v13
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v13
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v13
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v13
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v13
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v13
+; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v9, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e64 v10, v0, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e64 v0, v0, s5
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v13
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_lshlrev_b32 v0, 4, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s9
-; GFX11-NEXT: v_dual_mov_b32 v3, s10 :: v_dual_mov_b32 v4, s11
-; GFX11-NEXT: v_dual_mov_b32 v5, s12 :: v_dual_mov_b32 v6, s13
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e32 v9, v1, v2, vcc_lo
-; GFX11-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v8, s15
-; GFX11-NEXT: v_not_b32_e32 v10, v10
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v3, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, s9
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v0, v0, s5
+; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, s8, v2, vcc_lo
+; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s10, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s11, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v4, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v5, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s12, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s13, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v6, s3
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v7, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s14, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s15, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v8, s5
-; GFX11-NEXT: v_and_or_b32 v14, v9, v10, v0
-; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
-; GFX11-NEXT: v_mov_b32_e32 v12, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v14, s6
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v14, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v14, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, v14, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v5, v14, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v14, s3
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v7, v14, s4
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v8, v14, s5
+; GFX11-NEXT: v_and_or_b32 v7, v2, v3, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, s8, v7, s6
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s9, v7, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s10, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, s11, v7, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, s12, v7, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, s13, v7, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v6, s14, v7, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v7, s15, v7, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[9:10], v[0:3], off
-; GFX11-NEXT: global_store_b128 v[11:12], v[4:7], off
+; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[10:11], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -3575,106 +3554,93 @@ define amdgpu_ps void @insertelement_s_v16i16_v_v(ptr addrspace(4) inreg %ptr, i
;
; GFX10-LABEL: insertelement_s_v16i16_v_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x0
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 1, v1
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 1, v1
; GFX10-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX10-NEXT: v_mov_b32_e32 v12, 16
-; GFX10-NEXT: v_mov_b32_e32 v13, 0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: v_mov_b32_e32 v9, 0
+; GFX10-NEXT: v_mov_b32_e32 v10, 16
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v2
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX10-NEXT: v_lshlrev_b32_e64 v11, v1, 0xffff
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_not_b32_e32 v1, v11
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v2
; GFX10-NEXT: v_mov_b32_e32 v11, 0
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v1, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mov_b32_e32 v9, s7
-; GFX10-NEXT: v_mov_b32_e32 v8, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v10, v2, v3, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v14
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v14
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v14
-; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v14
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v14
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v4, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v14
-; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v14
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v5, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v7, s3
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v9, s5
-; GFX10-NEXT: v_and_or_b32 v15, v10, v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v10, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v15, s6
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v15, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v15, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v15, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v15, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v7, v15, s3
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v8, v15, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, v15, s5
-; GFX10-NEXT: global_store_dwordx4 v[10:11], v[0:3], off
-; GFX10-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX10-NEXT: v_mov_b32_e32 v3, s9
+; GFX10-NEXT: v_not_b32_e32 v1, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s8, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s10, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s11, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s12, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s13, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s14, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s15, s5
+; GFX10-NEXT: v_and_or_b32 v7, v3, v1, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, s8, v7, s6
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s9, v7, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s10, v7, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, s11, v7, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v4, s12, v7, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v5, s13, v7, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v6, s14, v7, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, s15, v7, s5
+; GFX10-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i16_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b256 s[0:7], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_mov_b32_e32 v13, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14
+; GFX11-NEXT: s_load_b256 s[8:15], s[2:3], 0x0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v1
+; GFX11-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_and_b32 v1, 1, v1
+; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e64 v11, v1, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_not_b32_e32 v1, v11
-; GFX11-NEXT: v_mov_b32_e32 v11, 0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v2
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v1, 0xffff
+; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v2
+; GFX11-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_and_b32 v0, 0xffff, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
-; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
-; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v10, v2, v3, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v14
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v14
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v14
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v14
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v14
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v4, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v14
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v5, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_lshlrev_b32 v0, v1, v0
+; GFX11-NEXT: v_not_b32_e32 v1, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s8, v3, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v7, s3
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s10, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s11, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v9, s5
-; GFX11-NEXT: v_and_or_b32 v15, v10, v1, v0
-; GFX11-NEXT: v_mov_b32_e32 v10, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_cndmask_b32 v1, v3, v15
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v15, s6
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v15, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, v15, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v15, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v7, v15, s3
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v8, v15, s4
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v9, v15, s5
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s12, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s13, s3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s14, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s15, s5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v7, v3, v1, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, s8, v7, s6
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s9, v7, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s10, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, s11, v7, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, s12, v7, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, s13, v7, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v6, s14, v7, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v7, s15, v7, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[10:11], v[0:3], off
-; GFX11-NEXT: global_store_b128 v[12:13], v[4:7], off
+; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[10:11], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
index 620cba152f5a2..ce140334d66b9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
@@ -449,16 +449,15 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX10-NEXT: global_load_ushort v1, v1, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: s_lshr_b32 s1, s0, 8
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s1, v1, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_and_b32_sdwa v3, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s4, vcc_lo
+; GFX10-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_cndmask_b32_e32 v4, s0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: global_store_short v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
@@ -469,21 +468,21 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16-NEXT: global_load_u16 v1, v1, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, s1, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, v2, s4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, s0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
; GFX11-TRUE16-NEXT: s_endpgm
@@ -495,19 +494,18 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX11-FAKE16-NEXT: global_load_u16 v1, v1, s[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, s1, v1, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v2, s4, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, s0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xff, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v3
; GFX11-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-FAKE16-NEXT: s_endpgm
@@ -2098,51 +2096,49 @@ define amdgpu_ps void @insertelement_s_v8i8_s_v(ptr addrspace(4) inreg %ptr, i8
;
; GFX10-LABEL: insertelement_s_v8i8_s_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX10-NEXT: v_and_b32_e32 v1, 3, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 2, v0
-; GFX10-NEXT: s_and_b32 s2, s4, 0xff
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v1
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v2, 0xff
-; GFX10-NEXT: v_lshlrev_b32_e64 v2, v2, s2
-; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 2, v0
+; GFX10-NEXT: s_and_b32 s0, s4, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v1, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX10-NEXT: v_not_b32_e32 v2, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX10-NEXT: v_and_or_b32 v5, v5, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: v_mov_b32_e32 v0, s3
+; GFX10-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v4, v0, v2, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s2, v4, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v4, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i8_s_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v1, 3, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 2, v0
-; GFX11-NEXT: s_and_b32 s2, s4, 0xff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 2, v0
+; GFX11-NEXT: s_and_b32 s0, s4, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 3, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v2, 0xff
-; GFX11-NEXT: v_lshlrev_b32_e64 v2, v2, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_lshlrev_b32 v1, 3, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v1, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX11-NEXT: v_not_b32_e32 v2, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_or_b32 v5, v5, v3, v2
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: v_and_or_b32 v4, v0, v2, v1
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s2, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s3, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i8> %vec, i8 %val, i32 %idx
@@ -2226,50 +2222,48 @@ define amdgpu_ps void @insertelement_s_v8i8_v_v(ptr addrspace(4) inreg %ptr, i8
;
; GFX10-LABEL: insertelement_s_v8i8_v_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX10-NEXT: v_and_b32_e32 v2, 3, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 2, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v2
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
-; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xff
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_not_b32_e32 v3, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v2, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_not_b32_e32 v2, v3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc_lo
-; GFX10-NEXT: v_and_or_b32 v6, v4, v3, v0
-; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v6, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
-; GFX10-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v3, v1, v2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s2, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v3, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i8_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v2, 3, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 2, v1
-; GFX11-NEXT: v_and_b32_e32 v3, 0xff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 3, v2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v2, 0xff
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, v2, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v3, v4
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
-; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_lshlrev_b32 v2, 3, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v2, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_not_b32_e32 v2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v3, v1, v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s2, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s3, v3, vcc_lo
+; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i8> %vec, i8 %val, i32 %idx
@@ -3085,65 +3079,61 @@ define amdgpu_ps void @insertelement_s_v16i8_s_v(ptr addrspace(4) inreg %ptr, i8
;
; GFX10-LABEL: insertelement_s_v16i8_s_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 2, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 3, v0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xff
-; GFX10-NEXT: v_not_b32_e32 v7, v7
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX10-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v1, 3, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_and_b32 s1, s4, 0xff
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_e64 v4, v4, s1
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
-; GFX10-NEXT: v_and_or_b32 v7, v5, v7, v4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v1, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v1, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s9
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s8, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s10, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s11, s1
+; GFX10-NEXT: v_and_or_b32 v3, v2, v3, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v0, s8, v3, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s9, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s10, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, s11, v3, s1
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i8_s_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 2, v0
-; GFX11-NEXT: v_and_b32_e32 v4, 3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX11-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v1, 3, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_and_b32 s1, s4, 0xff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
-; GFX11-NEXT: v_lshlrev_b32_e64 v7, v4, 0xff
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v4, s1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_not_b32_e32 v7, v7
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_or_b32 v7, v5, v7, v4
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v2, s9 :: v_dual_lshlrev_b32 v1, 3, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v2, s8, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v1, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v1, s1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s10, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s11, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v3, v2, v3, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, s8, v3, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s9, v3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s10, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, s11, v3, s1
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(4) %ptr
@@ -3252,66 +3242,63 @@ define amdgpu_ps void @insertelement_s_v16i8_v_v(ptr addrspace(4) inreg %ptr, i8
;
; GFX10-LABEL: insertelement_s_v16i8_v_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 2, v1
-; GFX10-NEXT: v_and_b32_e32 v5, 3, v1
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v5
-; GFX10-NEXT: v_lshlrev_b32_e64 v8, v5, 0xff
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_not_b32_e32 v5, v8
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
-; GFX10-NEXT: v_and_or_b32 v8, v6, v5, v0
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v2, 3, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 2, v1
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_mov_b32_e32 v6, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
-; GFX10-NEXT: global_store_dwordx4 v[5:6], v[0:3], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: s_mov_b32 null, 0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v2, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_not_b32_e32 v2, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s4, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s6, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s7, s1
+; GFX10-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, s4, v3, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s5, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, s6, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, s7, v3, s1
+; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i8_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 2, v1
-; GFX11-NEXT: v_and_b32_e32 v5, 3, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
+; GFX11-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v2, 3, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 2, v1
+; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0xff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v4, s3
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v5
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
-; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xff
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, v5, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_not_b32_e32 v5, v8
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_or_b32 v8, v6, v5, v0
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
-; GFX11-NEXT: global_store_b128 v[5:6], v[0:3], off
+; GFX11-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_lshlrev_b32 v2, 3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s4, v3, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v2, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s6, s0
+; GFX11-NEXT: v_not_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s7, s1
+; GFX11-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, s4, v3, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s5, v3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, s6, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, s7, v3, s1
+; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index a134e02dc9b51..d977913e4d89b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -239,57 +239,45 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_s_s_v(<8 x float> inreg %v
;
; GFX10-LABEL: dyn_insertelement_v8f32_s_s_v:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: v_mov_b32_e32 v7, s10
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mov_b32_e32 v3, s4
-; GFX10-NEXT: v_mov_b32_e32 v4, s5
-; GFX10-NEXT: v_mov_b32_e32 v5, s6
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v1, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v8, s2, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT: v_mov_b32_e32 v6, s7
-; GFX10-NEXT: v_mov_b32_e32 v7, s8
-; GFX10-NEXT: v_mov_b32_e32 v9, s9
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v2, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v1, s3, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s4, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s5, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v5, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v4, s6, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v6, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v5, s7, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v7, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v6, s8, v7, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
; GFX10-NEXT: v_mov_b32_e32 v0, v8
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, s10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v7, s9, v7, vcc_lo
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: dyn_insertelement_v8f32_s_s_v:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-NEXT: v_mov_b32_e32 v7, s10
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s5
-; GFX11-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s7
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v1, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v8, s2, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT: v_mov_b32_e32 v7, s8
-; GFX11-NEXT: v_mov_b32_e32 v9, s9
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v2, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v1, s3, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v2, s4, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s5, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v5, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v4, s6, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v5, s7, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v7, s10, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v6, s8, v7, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, v8
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v9, s10, vcc_lo
+; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_cndmask_b32 v7, s9, v7
; GFX11-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: dyn_insertelement_v8f32_s_s_v:
@@ -838,59 +826,37 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
; GPRIDX-LABEL: dyn_insertelement_v8f64_const_s_v_v:
; GPRIDX: ; %bb.0: ; %entry
; GPRIDX-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GPRIDX-NEXT: s_mov_b32 s18, 0
-; GPRIDX-NEXT: s_mov_b32 s19, 0x40200000
-; GPRIDX-NEXT: s_mov_b32 s16, 0
-; GPRIDX-NEXT: s_mov_b32 s17, 0x401c0000
-; GPRIDX-NEXT: s_mov_b32 s14, 0
-; GPRIDX-NEXT: s_mov_b32 s15, 0x40180000
-; GPRIDX-NEXT: s_mov_b32 s12, 0
-; GPRIDX-NEXT: s_mov_b32 s13, 0x40140000
-; GPRIDX-NEXT: s_mov_b64 s[10:11], 4.0
-; GPRIDX-NEXT: s_mov_b32 s8, 0
-; GPRIDX-NEXT: s_mov_b32 s9, 0x40080000
-; GPRIDX-NEXT: s_mov_b64 s[6:7], 2.0
-; GPRIDX-NEXT: s_mov_b64 s[4:5], 1.0
-; GPRIDX-NEXT: v_mov_b32_e32 v3, s4
-; GPRIDX-NEXT: v_mov_b32_e32 v4, s5
-; GPRIDX-NEXT: v_mov_b32_e32 v5, s6
-; GPRIDX-NEXT: v_mov_b32_e32 v6, s7
-; GPRIDX-NEXT: v_mov_b32_e32 v7, s8
-; GPRIDX-NEXT: v_mov_b32_e32 v8, s9
-; GPRIDX-NEXT: v_mov_b32_e32 v9, s10
-; GPRIDX-NEXT: v_mov_b32_e32 v10, s11
-; GPRIDX-NEXT: v_mov_b32_e32 v11, s12
-; GPRIDX-NEXT: v_mov_b32_e32 v12, s13
-; GPRIDX-NEXT: v_mov_b32_e32 v13, s14
-; GPRIDX-NEXT: v_mov_b32_e32 v14, s15
-; GPRIDX-NEXT: v_mov_b32_e32 v15, s16
-; GPRIDX-NEXT: v_mov_b32_e32 v16, s17
-; GPRIDX-NEXT: v_mov_b32_e32 v17, s18
-; GPRIDX-NEXT: v_mov_b32_e32 v18, s19
+; GPRIDX-NEXT: v_mov_b32_e32 v4, 0x3ff00000
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GPRIDX-NEXT: v_mov_b32_e32 v5, 0x40080000
+; GPRIDX-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
+; GPRIDX-NEXT: v_mov_b32_e32 v6, 0x40100000
+; GPRIDX-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 3, v2
+; GPRIDX-NEXT: v_mov_b32_e32 v12, 0x40140000
+; GPRIDX-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v10, v6, v1, vcc
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
+; GPRIDX-NEXT: v_mov_b32_e32 v14, 0x40180000
+; GPRIDX-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v12, v12, v1, vcc
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
+; GPRIDX-NEXT: v_mov_b32_e32 v16, 0x401c0000
+; GPRIDX-NEXT: v_cndmask_b32_e32 v13, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v14, v14, v1, vcc
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 6, v2
+; GPRIDX-NEXT: v_cndmask_b32_e32 v15, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v16, v16, v1, vcc
; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[16:17], 0, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[4:5], 2, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[6:7], 3, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[8:9], 4, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[10:11], 5, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[12:13], 6, v2
-; GPRIDX-NEXT: v_cmp_eq_u32_e64 s[14:15], 7, v2
-; GPRIDX-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[16:17]
-; GPRIDX-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e64 v4, v4, v1, s[16:17]
-; GPRIDX-NEXT: v_cndmask_b32_e32 v6, v6, v1, vcc
-; GPRIDX-NEXT: v_cndmask_b32_e64 v7, v7, v0, s[4:5]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v9, v9, v0, s[6:7]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v11, v11, v0, s[8:9]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v13, v13, v0, s[10:11]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v15, v15, v0, s[12:13]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v17, v17, v0, s[14:15]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v10, v10, v1, s[6:7]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v12, v12, v1, s[8:9]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v14, v14, v1, s[10:11]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v16, v16, v1, s[12:13]
-; GPRIDX-NEXT: v_cndmask_b32_e64 v18, v18, v1, s[14:15]
+; GPRIDX-NEXT: v_mov_b32_e32 v18, 0x40200000
+; GPRIDX-NEXT: v_cndmask_b32_e32 v5, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v6, 2.0, v1, vcc
+; GPRIDX-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
+; GPRIDX-NEXT: v_cndmask_b32_e32 v17, 0, v0, vcc
+; GPRIDX-NEXT: v_cndmask_b32_e32 v18, v18, v1, vcc
; GPRIDX-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GPRIDX-NEXT: s_waitcnt vmcnt(0)
; GPRIDX-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
@@ -904,59 +870,30 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
; GFX10-LABEL: dyn_insertelement_v8f64_const_s_v_v:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b64 s[4:5], 1.0
-; GFX10-NEXT: s_mov_b32 s18, 0
-; GFX10-NEXT: s_mov_b32 s19, 0x40200000
-; GFX10-NEXT: s_mov_b32 s16, 0
-; GFX10-NEXT: s_mov_b32 s17, 0x401c0000
-; GFX10-NEXT: s_mov_b32 s14, 0
-; GFX10-NEXT: s_mov_b32 s15, 0x40180000
-; GFX10-NEXT: s_mov_b32 s12, 0
-; GFX10-NEXT: s_mov_b32 s13, 0x40140000
-; GFX10-NEXT: s_mov_b64 s[10:11], 4.0
-; GFX10-NEXT: s_mov_b32 s8, 0
-; GFX10-NEXT: s_mov_b32 s9, 0x40080000
-; GFX10-NEXT: s_mov_b64 s[6:7], 2.0
-; GFX10-NEXT: v_mov_b32_e32 v3, s4
-; GFX10-NEXT: v_mov_b32_e32 v4, s5
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s6
-; GFX10-NEXT: v_mov_b32_e32 v6, s7
-; GFX10-NEXT: v_mov_b32_e32 v7, s8
-; GFX10-NEXT: v_mov_b32_e32 v8, s9
-; GFX10-NEXT: v_mov_b32_e32 v9, s10
-; GFX10-NEXT: v_mov_b32_e32 v10, s11
-; GFX10-NEXT: v_mov_b32_e32 v11, s12
-; GFX10-NEXT: v_mov_b32_e32 v12, s13
-; GFX10-NEXT: v_mov_b32_e32 v13, s14
-; GFX10-NEXT: v_mov_b32_e32 v14, s15
-; GFX10-NEXT: v_mov_b32_e32 v15, s16
-; GFX10-NEXT: v_mov_b32_e32 v16, s17
-; GFX10-NEXT: v_mov_b32_e32 v17, s18
-; GFX10-NEXT: v_mov_b32_e32 v18, s19
; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 1, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v0, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v1, s4
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 3, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 2, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 3, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v4, 0x3ff00000, v1, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v17, v17, v0, s5
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v0, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v1, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, v0, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v6, 2.0, v1, s4
; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 5, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v12, v12, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, v0, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v8, 0x40080000, v1, s5
+; GFX10-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v12, 0x40140000, v1, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v18, v1, s5
-; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v0, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v1, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v2
+; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, v0, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v10, 0x40100000, v1, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, v0, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v14, 0x40180000, v1, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v15, 0, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v16, 0x401c0000, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v17, 0, v0, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v18, 0x40200000, v1, s5
; GFX10-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
@@ -970,47 +907,27 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
; GFX11-LABEL: dyn_insertelement_v8f64_const_s_v_v:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b64 s[0:1], 1.0
-; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_mov_b32 s15, 0x40200000
-; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_mov_b32 s13, 0x401c0000
-; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_mov_b32 s11, 0x40180000
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_mov_b32 s9, 0x40140000
-; GFX11-NEXT: s_mov_b64 s[6:7], 4.0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s5, 0x40080000
-; GFX11-NEXT: s_mov_b64 s[2:3], 2.0
-; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-NEXT: v_dual_mov_b32 v18, s15 :: v_dual_mov_b32 v17, s14
-; GFX11-NEXT: v_dual_mov_b32 v16, s13 :: v_dual_mov_b32 v15, s12
-; GFX11-NEXT: v_dual_mov_b32 v14, s11 :: v_dual_mov_b32 v13, s10
-; GFX11-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v11, s8
-; GFX11-NEXT: v_dual_mov_b32 v10, s7 :: v_dual_mov_b32 v9, s6
-; GFX11-NEXT: v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v7, s4
-; GFX11-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v5, s2
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX11-NEXT: v_dual_cndmask_b32 v3, v3, v0 :: v_dual_cndmask_b32 v4, v4, v1
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 7, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX11-NEXT: v_dual_cndmask_b32 v7, v7, v0 :: v_dual_cndmask_b32 v8, v8, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 2, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 3, v2
+; GFX11-NEXT: v_dual_cndmask_b32 v3, 0, v0 :: v_dual_cndmask_b32 v4, 0x3ff00000, v1
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v17, v17, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v1, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v6, 2.0, v1, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 5, v2
-; GFX11-NEXT: v_dual_cndmask_b32 v11, v11, v0 :: v_dual_cndmask_b32 v12, v12, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v7, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v8, 0x40080000, v1, s1
+; GFX11-NEXT: v_dual_cndmask_b32 v11, 0, v0 :: v_dual_cndmask_b32 v12, 0x40140000, v1
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v18, v18, v1, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v13, v13, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v14, v14, v1, s0
-; GFX11-NEXT: v_dual_cndmask_b32 v15, v15, v0 :: v_dual_cndmask_b32 v16, v16, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 7, v2
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v0, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v10, 0x40100000, v1, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v13, 0, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v14, 0x40180000, v1, s0
+; GFX11-NEXT: v_dual_cndmask_b32 v15, 0, v0 :: v_dual_cndmask_b32 v16, 0x401c0000, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v17, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v18, 0x40200000, v1, s1
; GFX11-NEXT: global_store_b128 v[0:1], v[3:6], off dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_store_b128 v[0:1], v[7:10], off dlc
@@ -1024,59 +941,37 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
; GFX8-LABEL: dyn_insertelement_v8f64_const_s_v_v:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s18, 0
-; GFX8-NEXT: s_mov_b32 s19, 0x40200000
-; GFX8-NEXT: s_mov_b32 s16, 0
-; GFX8-NEXT: s_mov_b32 s17, 0x401c0000
-; GFX8-NEXT: s_mov_b32 s14, 0
-; GFX8-NEXT: s_mov_b32 s15, 0x40180000
-; GFX8-NEXT: s_mov_b32 s12, 0
-; GFX8-NEXT: s_mov_b32 s13, 0x40140000
-; GFX8-NEXT: s_mov_b64 s[10:11], 4.0
-; GFX8-NEXT: s_mov_b32 s8, 0
-; GFX8-NEXT: s_mov_b32 s9, 0x40080000
-; GFX8-NEXT: s_mov_b64 s[6:7], 2.0
-; GFX8-NEXT: s_mov_b64 s[4:5], 1.0
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: v_mov_b32_e32 v7, s8
-; GFX8-NEXT: v_mov_b32_e32 v8, s9
-; GFX8-NEXT: v_mov_b32_e32 v9, s10
-; GFX8-NEXT: v_mov_b32_e32 v10, s11
-; GFX8-NEXT: v_mov_b32_e32 v11, s12
-; GFX8-NEXT: v_mov_b32_e32 v12, s13
-; GFX8-NEXT: v_mov_b32_e32 v13, s14
-; GFX8-NEXT: v_mov_b32_e32 v14, s15
-; GFX8-NEXT: v_mov_b32_e32 v15, s16
-; GFX8-NEXT: v_mov_b32_e32 v16, s17
-; GFX8-NEXT: v_mov_b32_e32 v17, s18
-; GFX8-NEXT: v_mov_b32_e32 v18, s19
+; GFX8-NEXT: v_mov_b32_e32 v4, 0x3ff00000
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, 0x40080000
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
+; GFX8-NEXT: v_mov_b32_e32 v6, 0x40100000
+; GFX8-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v5, v1, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v2
+; GFX8-NEXT: v_mov_b32_e32 v12, 0x40140000
+; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v6, v1, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
+; GFX8-NEXT: v_mov_b32_e32 v14, 0x40180000
+; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v12, v1, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
+; GFX8-NEXT: v_mov_b32_e32 v16, 0x401c0000
+; GFX8-NEXT: v_cndmask_b32_e32 v13, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v14, v1, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v15, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v16, v1, vcc
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 0, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 2, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 3, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 4, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 5, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 6, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 7, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[16:17]
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v1, s[16:17]
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v0, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v0, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v0, s[8:9]
-; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v0, s[10:11]
-; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v0, s[12:13]
-; GFX8-NEXT: v_cndmask_b32_e64 v17, v17, v0, s[14:15]
-; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v1, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v1, s[8:9]
-; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v1, s[10:11]
-; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v1, s[12:13]
-; GFX8-NEXT: v_cndmask_b32_e64 v18, v18, v1, s[14:15]
+; GFX8-NEXT: v_mov_b32_e32 v18, 0x40200000
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, 2.0, v1, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v17, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v18, v18, v1, vcc
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[7:10]
@@ -1583,46 +1478,30 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_v(<8 x double> inreg %vec, do
;
; GFX10-LABEL: dyn_insertelement_v8f64_s_v_v:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
-; GFX10-NEXT: v_mov_b32_e32 v6, s5
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX10-NEXT: v_mov_b32_e32 v7, s6
-; GFX10-NEXT: v_mov_b32_e32 v8, s7
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 7, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s2, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v4, s3, v1, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_mov_b32_e32 v9, s8
-; GFX10-NEXT: v_mov_b32_e32 v10, s9
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v5, s4, v0, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v6, s5, v1, s0
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX10-NEXT: v_mov_b32_e32 v11, s10
-; GFX10-NEXT: v_mov_b32_e32 v12, s11
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v17, s16, v0, s1
+; GFX10-NEXT: v_cndmask_b32_e32 v7, s6, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v8, s7, v1, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_mov_b32_e32 v13, s12
-; GFX10-NEXT: v_mov_b32_e32 v14, s13
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v1, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v9, s8, v0, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v10, s9, v1, s0
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 5, v2
-; GFX10-NEXT: v_mov_b32_e32 v15, s14
-; GFX10-NEXT: v_mov_b32_e32 v16, s15
-; GFX10-NEXT: v_mov_b32_e32 v17, s16
-; GFX10-NEXT: v_mov_b32_e32 v18, s17
-; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v12, v12, v1, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v18, s17, v1, s1
+; GFX10-NEXT: v_cndmask_b32_e32 v11, s10, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v12, s11, v1, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 7, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v1, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v17, v17, v0, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v18, v1, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v13, s12, v0, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v14, s13, v1, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v15, s14, v0, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v16, s15, v1, vcc_lo
; GFX10-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
@@ -1635,34 +1514,30 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_v(<8 x double> inreg %vec, do
;
; GFX11-LABEL: dyn_insertelement_v8f64_s_v_v:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s3
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX11-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v8, s7
-; GFX11-NEXT: v_dual_cndmask_b32 v3, v3, v0 :: v_dual_cndmask_b32 v4, v4, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 7, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v3, s2, v0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v4, s3, v1, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-NEXT: v_dual_mov_b32 v9, s8 :: v_dual_mov_b32 v10, s9
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v5, s4, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v6, s5, v1, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX11-NEXT: v_dual_mov_b32 v11, s10 :: v_dual_mov_b32 v12, s11
-; GFX11-NEXT: v_dual_cndmask_b32 v7, v7, v0 :: v_dual_cndmask_b32 v8, v8, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v17, s16, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v7, s6, v0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v8, s7, v1, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX11-NEXT: v_dual_mov_b32 v13, s12 :: v_dual_mov_b32 v14, s13
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v1, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v9, s8, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v10, s9, v1, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 5, v2
-; GFX11-NEXT: v_dual_mov_b32 v15, s14 :: v_dual_mov_b32 v16, s15
-; GFX11-NEXT: v_dual_mov_b32 v17, s16 :: v_dual_mov_b32 v18, s17
-; GFX11-NEXT: v_dual_cndmask_b32 v11, v11, v0 :: v_dual_cndmask_b32 v12, v12, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v18, s17, v1, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v11, s10, v0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v12, s11, v1, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 7, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v13, v13, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v14, v14, v1, s0
-; GFX11-NEXT: v_dual_cndmask_b32 v15, v15, v0 :: v_dual_cndmask_b32 v16, v16, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v17, v17, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v18, v18, v1, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v13, s12, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v14, s13, v1, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v15, s14, v0, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v16, s15, v1, vcc_lo
; GFX11-NEXT: global_store_b128 v[0:1], v[3:6], off dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_store_b128 v[0:1], v[7:10], off dlc
@@ -6529,101 +6404,44 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
; GPRIDX-NEXT: v_readfirstlane_b32 s13, v1
; GPRIDX-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: dyn_insertelement_v7f64_s_v_v:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
-; GFX10-NEXT: v_mov_b32_e32 v6, s5
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX10-NEXT: v_mov_b32_e32 v7, s6
-; GFX10-NEXT: v_mov_b32_e32 v8, s7
-; GFX10-NEXT: v_mov_b32_e32 v9, s8
-; GFX10-NEXT: v_mov_b32_e32 v10, s9
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX10-NEXT: v_mov_b32_e32 v11, s10
-; GFX10-NEXT: v_mov_b32_e32 v12, s11
-; GFX10-NEXT: v_mov_b32_e32 v13, s12
-; GFX10-NEXT: v_mov_b32_e32 v14, s13
-; GFX10-NEXT: v_mov_b32_e32 v15, s14
-; GFX10-NEXT: v_mov_b32_e32 v16, s15
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v1, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 5, v2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 6, v2
-; GFX10-NEXT: v_readfirstlane_b32 s2, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v11, v11, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v12, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v12, v13, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v13, v14, v1, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v15, v0, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v16, v1, s1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v3
-; GFX10-NEXT: v_readfirstlane_b32 s1, v4
-; GFX10-NEXT: v_readfirstlane_b32 s3, v6
-; GFX10-NEXT: v_readfirstlane_b32 s4, v7
-; GFX10-NEXT: v_readfirstlane_b32 s5, v8
-; GFX10-NEXT: v_readfirstlane_b32 s6, v9
-; GFX10-NEXT: v_readfirstlane_b32 s7, v10
-; GFX10-NEXT: v_readfirstlane_b32 s8, v11
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v12
-; GFX10-NEXT: v_readfirstlane_b32 s11, v13
-; GFX10-NEXT: v_readfirstlane_b32 s12, v0
-; GFX10-NEXT: v_readfirstlane_b32 s13, v1
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: dyn_insertelement_v7f64_s_v_v:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s3
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v6, s5
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX11-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v8, s7
-; GFX11-NEXT: v_dual_mov_b32 v9, s8 :: v_dual_mov_b32 v10, s9
-; GFX11-NEXT: v_dual_cndmask_b32 v3, v3, v0 :: v_dual_cndmask_b32 v4, v4, v1
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX11-NEXT: v_dual_mov_b32 v11, s10 :: v_dual_mov_b32 v12, s11
-; GFX11-NEXT: v_dual_mov_b32 v13, s12 :: v_dual_mov_b32 v14, s13
-; GFX11-NEXT: v_dual_mov_b32 v15, s14 :: v_dual_mov_b32 v16, s15
-; GFX11-NEXT: v_dual_cndmask_b32 v7, v7, v0 :: v_dual_cndmask_b32 v8, v8, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v1, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 5, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s2, v5
-; GFX11-NEXT: v_dual_cndmask_b32 v11, v11, v0 :: v_dual_cndmask_b32 v2, v12, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v12, v13, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v13, v14, v1, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v15, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v16, v1, s1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: v_readfirstlane_b32 s1, v4
-; GFX11-NEXT: v_readfirstlane_b32 s3, v6
-; GFX11-NEXT: v_readfirstlane_b32 s4, v7
-; GFX11-NEXT: v_readfirstlane_b32 s5, v8
-; GFX11-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-NEXT: v_readfirstlane_b32 s7, v10
-; GFX11-NEXT: v_readfirstlane_b32 s8, v11
-; GFX11-NEXT: v_readfirstlane_b32 s9, v2
-; GFX11-NEXT: v_readfirstlane_b32 s10, v12
-; GFX11-NEXT: v_readfirstlane_b32 s11, v13
-; GFX11-NEXT: v_readfirstlane_b32 s12, v0
-; GFX11-NEXT: v_readfirstlane_b32 s13, v1
-; GFX11-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: dyn_insertelement_v7f64_s_v_v:
+; GFX10PLUS: ; %bb.0: ; %entry
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s1, 6, v2
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v3, s2, v0, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v4, s3, v1, vcc_lo
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v5, s4, v0, s0
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v6, s5, v1, s0
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v7, s6, v0, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v8, s7, v1, vcc_lo
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v9, s8, v0, s0
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v10, s9, v1, s0
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s0, 5, v2
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v5
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v11, s10, v0, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v2, s11, v1, vcc_lo
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v6
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v12, s12, v0, s0
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v13, s13, v1, s0
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v0, s14, v0, s1
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v1, s15, v1, s1
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s4, v7
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s5, v8
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s6, v9
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s7, v10
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s8, v11
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s9, v2
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s10, v12
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s11, v13
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s12, v0
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s13, v1
+; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: dyn_insertelement_v7f64_s_v_v:
; GFX8: ; %bb.0: ; %entry
@@ -7110,76 +6928,34 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_s_v_v(<5 x double> inreg
; GPRIDX-NEXT: v_readfirstlane_b32 s9, v1
; GPRIDX-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: dyn_insertelement_v5f64_s_v_v:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, s3
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
-; GFX10-NEXT: v_mov_b32_e32 v6, s5
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX10-NEXT: v_mov_b32_e32 v7, s6
-; GFX10-NEXT: v_mov_b32_e32 v8, s7
-; GFX10-NEXT: v_mov_b32_e32 v9, s8
-; GFX10-NEXT: v_mov_b32_e32 v10, s9
-; GFX10-NEXT: v_mov_b32_e32 v11, s10
-; GFX10-NEXT: v_mov_b32_e32 v12, s11
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 4, v2
-; GFX10-NEXT: v_readfirstlane_b32 s2, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v8, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v9, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v10, v1, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v11, v0, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v12, v1, s1
-; GFX10-NEXT: v_readfirstlane_b32 s0, v3
-; GFX10-NEXT: v_readfirstlane_b32 s1, v4
-; GFX10-NEXT: v_readfirstlane_b32 s3, v6
-; GFX10-NEXT: v_readfirstlane_b32 s4, v7
-; GFX10-NEXT: v_readfirstlane_b32 s5, v2
-; GFX10-NEXT: v_readfirstlane_b32 s6, v8
-; GFX10-NEXT: v_readfirstlane_b32 s7, v9
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: dyn_insertelement_v5f64_s_v_v:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s3
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v6, s5
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
-; GFX11-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v8, s7
-; GFX11-NEXT: v_dual_mov_b32 v9, s8 :: v_dual_mov_b32 v10, s9
-; GFX11-NEXT: v_dual_mov_b32 v11, s10 :: v_dual_mov_b32 v12, s11
-; GFX11-NEXT: v_dual_cndmask_b32 v3, v3, v0 :: v_dual_cndmask_b32 v4, v4, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v2
-; GFX11-NEXT: v_readfirstlane_b32 s2, v5
-; GFX11-NEXT: v_dual_cndmask_b32 v7, v7, v0 :: v_dual_cndmask_b32 v2, v8, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v9, v0, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v9, v10, v1, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v11, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v12, v1, s1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: v_readfirstlane_b32 s1, v4
-; GFX11-NEXT: v_readfirstlane_b32 s3, v6
-; GFX11-NEXT: v_readfirstlane_b32 s4, v7
-; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: v_readfirstlane_b32 s6, v8
-; GFX11-NEXT: v_readfirstlane_b32 s7, v9
-; GFX11-NEXT: v_readfirstlane_b32 s8, v0
-; GFX11-NEXT: v_readfirstlane_b32 s9, v1
-; GFX11-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: dyn_insertelement_v5f64_s_v_v:
+; GFX10PLUS: ; %bb.0: ; %entry
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s0, 1, v2
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s1, 4, v2
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v3, s2, v0, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v4, s3, v1, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v5, s4, v0, s0
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v6, s5, v1, s0
+; GFX10PLUS-NEXT: v_cmp_eq_u32_e64 s0, 3, v2
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s2, v5
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v7, s6, v0, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e32 v2, s7, v1, vcc_lo
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v8, s8, v0, s0
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v9, s9, v1, s0
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v0, s10, v0, s1
+; GFX10PLUS-NEXT: v_cndmask_b32_e64 v1, s11, v1, s1
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s3, v6
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s4, v7
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s6, v8
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s7, v9
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s8, v0
+; GFX10PLUS-NEXT: v_readfirstlane_b32 s9, v1
+; GFX10PLUS-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: dyn_insertelement_v5f64_s_v_v:
; GFX8: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
index f5a88e87225b3..f3deff250636c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
@@ -1331,11 +1331,10 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, [
; GFX8-LABEL: test_div_fmas_f32_i1_phi_vcc:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x28
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 2, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
; GFX8-NEXT: flat_load_dwordx3 v[1:3], v[1:2]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll
index d1dedfae6b1c5..614f604e1f12b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll
@@ -194,11 +194,10 @@ define amdgpu_kernel void @test_div_scale_f64_1(ptr addrspace(1) %out, ptr addrs
; GFX8-LABEL: test_div_scale_f64_1:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 8, v0
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
@@ -283,11 +282,10 @@ define amdgpu_kernel void @test_div_scale_f64_2(ptr addrspace(1) %out, ptr addrs
; GFX8-LABEL: test_div_scale_f64_2:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 8, v0
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
@@ -1009,24 +1007,24 @@ define amdgpu_kernel void @test_div_scale_f32_all_scalar_1(ptr addrspace(1) %out
define amdgpu_kernel void @test_div_scale_f32_all_scalar_2(ptr addrspace(1) %out, [8 x i32], float %a, [8 x i32], float %b) {
; GFX7-LABEL: test_div_scale_f32_all_scalar_2:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dword s3, s[4:5], 0x1c
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0x13
+; GFX7-NEXT: s_load_dword s3, s[4:5], 0x13
+; GFX7-NEXT: s_load_dword s6, s[4:5], 0x1c
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_div_scale_f32 v0, s[4:5], s6, v0, s6
+; GFX7-NEXT: v_div_scale_f32 v0, s[4:5], v0, s6, v0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: test_div_scale_f32_all_scalar_2:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x70
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x4c
+; GFX8-NEXT: s_load_dword s0, s[4:5], 0x4c
+; GFX8-NEXT: s_load_dword s1, s[4:5], 0x70
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_div_scale_f32 v2, s[0:1], s1, v0, s1
+; GFX8-NEXT: v_div_scale_f32 v2, s[0:1], v0, s1, v0
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
@@ -1127,13 +1125,13 @@ define amdgpu_kernel void @test_div_scale_f64_all_scalar_1(ptr addrspace(1) %out
define amdgpu_kernel void @test_div_scale_f64_all_scalar_2(ptr addrspace(1) %out, [8 x i32], double %a, [8 x i32], double %b) {
; GFX7-LABEL: test_div_scale_f64_all_scalar_2:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x1d
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x13
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x13
+; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x1d
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s2
; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: v_div_scale_f64 v[0:1], s[2:3], s[6:7], v[0:1], s[6:7]
+; GFX7-NEXT: v_div_scale_f64 v[0:1], s[2:3], v[0:1], s[6:7], v[0:1]
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -1141,12 +1139,12 @@ define amdgpu_kernel void @test_div_scale_f64_all_scalar_2(ptr addrspace(1) %out
;
; GFX8-LABEL: test_div_scale_f64_all_scalar_2:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x74
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x4c
+; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x4c
+; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x74
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_div_scale_f64 v[0:1], s[0:1], s[2:3], v[0:1], s[2:3]
+; GFX8-NEXT: v_div_scale_f64 v[0:1], s[0:1], v[0:1], s[2:3], v[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, s1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 17f3364422ee1..4c49ad37ef1db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -950,7 +950,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1030-NEXT: s_clause 0x1
; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1030-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT: v_mov_b32_e32 v3, 0
; GFX1030-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1030-NEXT: v_mov_b32_e32 v5, 2.0
@@ -961,10 +961,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1030-NEXT: v_mov_b32_e32 v10, 0x40e00000
; GFX1030-NEXT: v_mov_b32_e32 v11, 0x41000000
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX1030-NEXT: flat_load_dword v2, v[0:1]
; GFX1030-NEXT: v_mov_b32_e32 v0, 0xb36211c7
; GFX1030-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -979,7 +977,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1013-NEXT: s_clause 0x1
; GFX1013-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1013-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1013-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1013-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v3, 0
; GFX1013-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1013-NEXT: v_mov_b32_e32 v5, 2.0
@@ -990,10 +988,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1013-NEXT: v_mov_b32_e32 v10, 0x40e00000
; GFX1013-NEXT: v_mov_b32_e32 v11, 0x41000000
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_mov_b32_e32 v0, s6
-; GFX1013-NEXT: v_mov_b32_e32 v1, s7
-; GFX1013-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1013-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX1013-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1013-NEXT: v_add_co_ci_u32_e64 v1, s4, s7, 0, s4
; GFX1013-NEXT: flat_load_dword v2, v[0:1]
; GFX1013-NEXT: v_mov_b32_e32 v0, 0xb36211c7
; GFX1013-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -1010,19 +1006,17 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT: v_bfrev_b32_e32 v10, 4.0
+; GFX11-NEXT: v_dual_mov_b32 v2, 2.0 :: v_dual_mov_b32 v5, 0x40a00000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mov_b32 v9, 0xb36211c7 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-NEXT: v_dual_mov_b32 v4, 4.0 :: v_dual_mov_b32 v7, 0x40e00000
; GFX11-NEXT: v_mov_b32_e32 v8, 0x41000000
; GFX11-NEXT: v_mov_b32_e32 v6, 0x40c00000
-; GFX11-NEXT: v_mov_b32_e32 v9, 0xb36211c7
-; GFX11-NEXT: v_mov_b32_e32 v5, 0x40a00000
; GFX11-NEXT: v_mov_b32_e32 v3, 0x40400000
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v2, 2.0
+; GFX11-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX11-NEXT: flat_load_b32 v11, v[0:1]
; GFX11-NEXT: v_dual_mov_b32 v1, 1.0 :: v_dual_mov_b32 v0, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1053,7 +1047,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1030-NEXT: s_clause 0x1
; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1030-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT: v_mov_b32_e32 v3, 0
; GFX1030-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1030-NEXT: v_mov_b32_e32 v5, 2.0
@@ -1061,10 +1055,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1030-NEXT: v_mov_b32_e32 v7, 0x46004500
; GFX1030-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX1030-NEXT: flat_load_dword v2, v[0:1]
; GFX1030-NEXT: v_mov_b32_e32 v0, 0xb36211c6
; GFX1030-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -1079,7 +1071,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-NEXT: s_clause 0x1
; GFX1013-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1013-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1013-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1013-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v3, 0
; GFX1013-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1013-NEXT: v_mov_b32_e32 v5, 2.0
@@ -1087,10 +1079,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-NEXT: v_mov_b32_e32 v7, 0x46004500
; GFX1013-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_mov_b32_e32 v0, s6
-; GFX1013-NEXT: v_mov_b32_e32 v1, s7
-; GFX1013-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1013-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX1013-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1013-NEXT: v_add_co_ci_u32_e64 v1, s4, s7, 0, s4
; GFX1013-NEXT: flat_load_dword v2, v[0:1]
; GFX1013-NEXT: v_mov_b32_e32 v0, 0xb36211c6
; GFX1013-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -1107,26 +1097,22 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT: s_movk_i32 s5, 0x4600
-; GFX11-TRUE16-NEXT: s_movk_i32 s4, 0x4800
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, 0x4200
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s4, 0x4500
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, 0xb36211c6 :: v_dual_mov_b32 v5, s4
; GFX11-TRUE16-NEXT: v_bfrev_b32_e32 v7, 4.0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s5
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, 0x4200
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 2.0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
+; GFX11-TRUE16-NEXT: s_movk_i32 s4, 0x4800
; GFX11-TRUE16-NEXT: s_movk_i32 s6, 0x4700
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, 0x4400
-; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 2.0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s6
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s4, 0x4500
; GFX11-TRUE16-NEXT: flat_load_b32 v8, v[0:1]
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, 0x4400
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, 0xb36211c6 :: v_dual_mov_b32 v5, s4
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, 1.0 :: v_dual_mov_b32 v0, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s6
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -1141,16 +1127,14 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, 0xb36211c6
; GFX11-FAKE16-NEXT: v_bfrev_b32_e32 v7, 4.0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, 0x42004600 :: v_dual_mov_b32 v2, 2.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, 0x45004800 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0x44004700
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, 0x42004600
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0x45004800
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 2.0
+; GFX11-FAKE16-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX11-FAKE16-NEXT: flat_load_b32 v8, v[0:1]
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 1.0 :: v_dual_mov_b32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
index 3054c485eee11..1a5659713ade6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll
@@ -378,13 +378,11 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset(ptr addrspace(1) %ptr, i
; GFX12-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %soffset
@@ -416,13 +414,11 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX12-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:1024
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 %soffset
@@ -455,13 +451,11 @@ define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX12-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:1024
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 256
@@ -498,13 +492,12 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX12-LABEL: mubuf_store_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 %voffset
@@ -542,13 +535,12 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX12-LABEL: mubuf_store_sgpr_ptr_vgpr_offset_offset4095:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:16380
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 %voffset
@@ -586,13 +578,12 @@ define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX12-LABEL: mubuf_store_sgpr_ptr_offset4095_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_store_b32 v[0:1], v2, off offset:16380
; GFX12-NEXT: s_endpgm
%gep0 = getelementptr i32, ptr addrspace(1) %ptr, i32 4095
@@ -979,10 +970,9 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(ptr addrspace(1) %ptr, i
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1017,10 +1007,9 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(ptr addrspace(
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:1024 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1056,10 +1045,9 @@ define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(ptr addrspace(
; GFX12-NEXT: s_ashr_i32 s3, s2, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_lshl_b64 s[0:1], s[2:3], 2
-; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:1024 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1097,12 +1085,11 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(ptr addrspace(1) inreg %
; GFX12-LABEL: mubuf_load_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1141,12 +1128,11 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(ptr addrspace
; GFX12-LABEL: mubuf_load_sgpr_ptr_vgpr_offset_offset4095:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:16380 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1185,12 +1171,11 @@ define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(ptr addrspace
; GFX12-LABEL: mubuf_load_sgpr_ptr_offset4095_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:16380 scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@@ -1406,13 +1391,12 @@ define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_vgpr_offset(ptr addrspace(1) in
; GFX12-LABEL: mubuf_atomicrmw_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_mov_b32_e32 v2, 2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX12-NEXT: v_mov_b32_e32 v2, 2
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_atomic_add_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
@@ -1637,14 +1621,13 @@ define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_vgpr_offset(ptr addrspace(1) inre
;
; GFX12-LABEL: mubuf_cmpxchg_sgpr_ptr_vgpr_offset:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, s2
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-NEXT: v_mov_b32_e32 v5, s3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v4, v0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v5, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index b593c8af83e34..69df36bef4ce2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -997,16 +997,15 @@ define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
; GFX7-LABEL: s_mul_i96:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mul_hi_u32 v1, s0, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: s_mul_i32 s5, s0, s5
-; GFX7-NEXT: v_readfirstlane_b32 s7, v0
; GFX7-NEXT: s_mul_i32 s8, s1, s4
-; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: v_readfirstlane_b32 s7, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, v2, s4
; GFX7-NEXT: s_add_u32 s5, s8, s5
; GFX7-NEXT: s_mul_i32 s2, s2, s3
-; GFX7-NEXT: v_mul_hi_u32 v0, v0, s3
+; GFX7-NEXT: v_mul_hi_u32 v0, s1, v0
; GFX7-NEXT: s_mul_i32 s6, s0, s3
; GFX7-NEXT: s_add_u32 s2, s2, s5
; GFX7-NEXT: s_mul_i32 s0, s0, s4
@@ -1023,16 +1022,15 @@ define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
; GFX8-LABEL: s_mul_i96:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: v_mul_hi_u32 v1, s0, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_mul_i32 s5, s0, s5
-; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: s_mul_i32 s8, s1, s4
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_readfirstlane_b32 s7, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v2, s4
; GFX8-NEXT: s_add_u32 s5, s8, s5
; GFX8-NEXT: s_mul_i32 s2, s2, s3
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s1, v0
; GFX8-NEXT: s_mul_i32 s6, s0, s3
; GFX8-NEXT: s_add_u32 s2, s2, s5
; GFX8-NEXT: s_mul_i32 s0, s0, s4
@@ -1263,26 +1261,24 @@ define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
; GFX7-LABEL: s_mul_i128:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_mul_hi_u32 v2, s1, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
+; GFX7-NEXT: v_mul_hi_u32 v3, s1, v3
+; GFX7-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, v2, s6
; GFX7-NEXT: s_mul_i32 s10, s0, s6
-; GFX7-NEXT: v_readfirstlane_b32 s9, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX7-NEXT: v_readfirstlane_b32 s13, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mul_hi_u32 v2, v2, s4
; GFX7-NEXT: s_mul_i32 s12, s1, s5
-; GFX7-NEXT: v_readfirstlane_b32 s11, v0
+; GFX7-NEXT: v_readfirstlane_b32 s13, v3
+; GFX7-NEXT: v_readfirstlane_b32 s11, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, s2, v0
; GFX7-NEXT: s_add_u32 s10, s12, s10
-; GFX7-NEXT: v_mul_hi_u32 v1, s0, v1
-; GFX7-NEXT: v_mov_b32_e32 v0, s1
; GFX7-NEXT: s_addc_u32 s11, s13, s11
; GFX7-NEXT: s_mul_i32 s12, s2, s4
-; GFX7-NEXT: v_readfirstlane_b32 s13, v2
+; GFX7-NEXT: v_readfirstlane_b32 s13, v1
+; GFX7-NEXT: v_mul_hi_u32 v1, v2, s5
; GFX7-NEXT: s_add_u32 s10, s12, s10
-; GFX7-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX7-NEXT: v_mul_hi_u32 v0, s1, v0
; GFX7-NEXT: s_addc_u32 s11, s13, s11
; GFX7-NEXT: s_mul_i32 s12, s0, s5
; GFX7-NEXT: v_readfirstlane_b32 s13, v1
@@ -1311,26 +1307,24 @@ define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
; GFX8-LABEL: s_mul_i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_mul_hi_u32 v2, s1, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-NEXT: v_mul_hi_u32 v3, s1, v3
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v2, s6
; GFX8-NEXT: s_mul_i32 s10, s0, s6
-; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s13, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mul_hi_u32 v2, v2, s4
; GFX8-NEXT: s_mul_i32 s12, s1, s5
-; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: v_readfirstlane_b32 s13, v3
+; GFX8-NEXT: v_readfirstlane_b32 s11, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s2, v0
; GFX8-NEXT: s_add_u32 s10, s12, s10
-; GFX8-NEXT: v_mul_hi_u32 v1, s0, v1
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
; GFX8-NEXT: s_addc_u32 s11, s13, s11
; GFX8-NEXT: s_mul_i32 s12, s2, s4
-; GFX8-NEXT: v_readfirstlane_b32 s13, v2
+; GFX8-NEXT: v_readfirstlane_b32 s13, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v2, s5
; GFX8-NEXT: s_add_u32 s10, s12, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX8-NEXT: v_mul_hi_u32 v0, s1, v0
; GFX8-NEXT: s_addc_u32 s11, s13, s11
; GFX8-NEXT: s_mul_i32 s12, s0, s5
; GFX8-NEXT: v_readfirstlane_b32 s13, v1
@@ -1746,85 +1740,81 @@ define i128 @v_mul_i128(i128 %num, i128 %den) {
define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
; GFX7-LABEL: s_mul_i256:
; GFX7: ; %bb.0:
+; GFX7-NEXT: v_mov_b32_e32 v4, s9
+; GFX7-NEXT: v_mul_hi_u32 v5, s1, v4
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s8
+; GFX7-NEXT: v_mul_hi_u32 v2, v0, v1
+; GFX7-NEXT: v_readfirstlane_b32 s21, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s2, v1
; GFX7-NEXT: s_mov_b32 s16, s0
-; GFX7-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-NEXT: v_mul_hi_u32 v0, s16, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_mul_hi_u32 v2, s1, v1
-; GFX7-NEXT: v_mul_hi_u32 v1, s16, v1
-; GFX7-NEXT: v_readfirstlane_b32 s17, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s10
-; GFX7-NEXT: v_mul_hi_u32 v0, s16, v0
-; GFX7-NEXT: v_readfirstlane_b32 s21, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mul_hi_u32 v3, v2, s8
+; GFX7-NEXT: v_readfirstlane_b32 s17, v2
+; GFX7-NEXT: v_mul_hi_u32 v2, v0, s10
+; GFX7-NEXT: v_readfirstlane_b32 s22, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v0, v4
; GFX7-NEXT: s_mul_i32 s18, s16, s10
; GFX7-NEXT: s_mul_i32 s20, s1, s9
-; GFX7-NEXT: v_readfirstlane_b32 s19, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: v_readfirstlane_b32 s19, v2
; GFX7-NEXT: s_add_u32 s18, s20, s18
; GFX7-NEXT: s_addc_u32 s19, s21, s19
; GFX7-NEXT: s_mul_i32 s21, s2, s8
-; GFX7-NEXT: v_readfirstlane_b32 s23, v1
-; GFX7-NEXT: v_mul_hi_u32 v1, v0, s8
+; GFX7-NEXT: v_readfirstlane_b32 s23, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s1, v1
; GFX7-NEXT: s_cselect_b32 s20, 1, 0
-; GFX7-NEXT: v_readfirstlane_b32 s22, v3
; GFX7-NEXT: s_add_u32 s18, s21, s18
; GFX7-NEXT: s_addc_u32 s19, s22, s19
; GFX7-NEXT: s_mul_i32 s22, s16, s9
; GFX7-NEXT: s_cselect_b32 s21, 1, 0
; GFX7-NEXT: s_add_u32 s17, s22, s17
-; GFX7-NEXT: s_addc_u32 s22, s23, s18
-; GFX7-NEXT: v_readfirstlane_b32 s23, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s12
-; GFX7-NEXT: v_mul_hi_u32 v3, s16, v1
-; GFX7-NEXT: s_mul_i32 s18, s1, s8
+; GFX7-NEXT: s_addc_u32 s18, s23, s18
+; GFX7-NEXT: v_readfirstlane_b32 s23, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v0, s12
+; GFX7-NEXT: s_mul_i32 s22, s1, s8
+; GFX7-NEXT: v_mov_b32_e32 v6, s2
; GFX7-NEXT: s_cselect_b32 s25, 1, 0
-; GFX7-NEXT: s_add_u32 s18, s18, s17
-; GFX7-NEXT: s_addc_u32 s17, s23, s22
-; GFX7-NEXT: v_mov_b32_e32 v4, s11
-; GFX7-NEXT: v_readfirstlane_b32 s23, v3
-; GFX7-NEXT: v_mul_hi_u32 v3, v2, s10
-; GFX7-NEXT: v_mul_hi_u32 v5, s1, v4
+; GFX7-NEXT: s_add_u32 s17, s22, s17
+; GFX7-NEXT: s_addc_u32 s18, s23, s18
+; GFX7-NEXT: v_readfirstlane_b32 s23, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v6, s10
+; GFX7-NEXT: v_mov_b32_e32 v2, s1
+; GFX7-NEXT: v_mul_hi_u32 v7, v2, s11
; GFX7-NEXT: s_mul_i32 s22, s16, s12
+; GFX7-NEXT: v_readfirstlane_b32 s28, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s3, v4
; GFX7-NEXT: s_mul_i32 s24, s1, s11
-; GFX7-NEXT: v_readfirstlane_b32 s28, v3
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_readfirstlane_b32 s27, v5
-; GFX7-NEXT: v_mul_hi_u32 v5, v3, s9
; GFX7-NEXT: s_cselect_b32 s26, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s27, v7
+; GFX7-NEXT: v_readfirstlane_b32 s29, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s4, v1
; GFX7-NEXT: s_add_u32 s24, s24, s22
; GFX7-NEXT: s_addc_u32 s23, s27, s23
-; GFX7-NEXT: v_readfirstlane_b32 s29, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s4
-; GFX7-NEXT: v_mul_hi_u32 v6, v5, s8
; GFX7-NEXT: s_mul_i32 s27, s2, s10
+; GFX7-NEXT: v_readfirstlane_b32 s30, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v0, s11
; GFX7-NEXT: s_cselect_b32 s22, 1, 0
; GFX7-NEXT: s_add_u32 s24, s27, s24
-; GFX7-NEXT: v_mul_hi_u32 v0, v0, s10
; GFX7-NEXT: s_addc_u32 s27, s28, s23
+; GFX7-NEXT: v_readfirstlane_b32 s31, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v2, s10
; GFX7-NEXT: s_mul_i32 s28, s3, s9
; GFX7-NEXT: s_cselect_b32 s23, 1, 0
; GFX7-NEXT: s_add_u32 s28, s28, s24
-; GFX7-NEXT: v_readfirstlane_b32 s30, v6
-; GFX7-NEXT: v_mul_hi_u32 v6, s16, v4
; GFX7-NEXT: s_addc_u32 s27, s29, s27
; GFX7-NEXT: s_mul_i32 s29, s4, s8
; GFX7-NEXT: s_cselect_b32 s24, 1, 0
; GFX7-NEXT: s_add_u32 s28, s29, s28
-; GFX7-NEXT: v_readfirstlane_b32 s33, v0
-; GFX7-NEXT: v_mul_hi_u32 v0, v2, s9
+; GFX7-NEXT: v_readfirstlane_b32 s33, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s2, v4
; GFX7-NEXT: s_addc_u32 s27, s30, s27
; GFX7-NEXT: s_mul_i32 s30, s16, s11
; GFX7-NEXT: s_cselect_b32 s29, 1, 0
-; GFX7-NEXT: v_readfirstlane_b32 s31, v6
; GFX7-NEXT: s_add_u32 s19, s30, s19
; GFX7-NEXT: s_addc_u32 s28, s31, s28
; GFX7-NEXT: s_mul_i32 s31, s1, s10
; GFX7-NEXT: s_cselect_b32 s30, 1, 0
; GFX7-NEXT: s_add_u32 s19, s31, s19
-; GFX7-NEXT: v_readfirstlane_b32 s34, v0
-; GFX7-NEXT: v_mul_hi_u32 v0, v3, s8
+; GFX7-NEXT: v_readfirstlane_b32 s34, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s3, v1
; GFX7-NEXT: s_addc_u32 s28, s33, s28
; GFX7-NEXT: s_mul_i32 s33, s2, s9
; GFX7-NEXT: s_cselect_b32 s31, 1, 0
@@ -1832,87 +1822,85 @@ define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
; GFX7-NEXT: s_addc_u32 s28, s34, s28
; GFX7-NEXT: s_mul_i32 s34, s3, s8
; GFX7-NEXT: s_cselect_b32 s33, 1, 0
-; GFX7-NEXT: v_readfirstlane_b32 s35, v0
+; GFX7-NEXT: v_readfirstlane_b32 s35, v5
; GFX7-NEXT: s_add_u32 s19, s34, s19
; GFX7-NEXT: s_addc_u32 s28, s35, s28
; GFX7-NEXT: s_cselect_b32 s34, 1, 0
; GFX7-NEXT: s_cmp_lg_u32 s25, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s14
; GFX7-NEXT: s_cselect_b32 s25, 1, 0
; GFX7-NEXT: s_cmp_lg_u32 s26, 0
-; GFX7-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX7-NEXT: v_mul_hi_u32 v5, v0, s14
; GFX7-NEXT: s_addc_u32 s19, s25, s19
; GFX7-NEXT: s_cselect_b32 s25, 1, 0
; GFX7-NEXT: s_cmp_lg_u32 s20, 0
-; GFX7-NEXT: v_mov_b32_e32 v2, s13
; GFX7-NEXT: s_cselect_b32 s20, 1, 0
; GFX7-NEXT: s_cmp_lg_u32 s21, 0
-; GFX7-NEXT: v_mul_hi_u32 v6, s1, v2
+; GFX7-NEXT: v_mul_hi_u32 v8, v2, s13
; GFX7-NEXT: s_addc_u32 s20, s20, 0
-; GFX7-NEXT: v_readfirstlane_b32 s26, v0
-; GFX7-NEXT: v_mul_hi_u32 v0, s2, v1
+; GFX7-NEXT: v_readfirstlane_b32 s26, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v6, s12
; GFX7-NEXT: s_cmp_lg_u32 s25, 0
; GFX7-NEXT: s_addc_u32 s20, s20, s28
; GFX7-NEXT: s_mul_i32 s25, s16, s14
; GFX7-NEXT: s_mul_i32 s28, s1, s13
+; GFX7-NEXT: v_mov_b32_e32 v7, s3
; GFX7-NEXT: s_cselect_b32 s21, 1, 0
-; GFX7-NEXT: v_readfirstlane_b32 s35, v6
+; GFX7-NEXT: v_readfirstlane_b32 s35, v8
; GFX7-NEXT: s_add_u32 s25, s28, s25
; GFX7-NEXT: s_addc_u32 s26, s35, s26
-; GFX7-NEXT: v_readfirstlane_b32 s35, v0
-; GFX7-NEXT: v_mul_hi_u32 v0, v3, s11
+; GFX7-NEXT: v_readfirstlane_b32 s35, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, v7, s11
; GFX7-NEXT: s_mul_i32 s28, s2, s12
+; GFX7-NEXT: v_mov_b32_e32 v3, s10
; GFX7-NEXT: s_add_u32 s25, s28, s25
; GFX7-NEXT: s_addc_u32 s26, s35, s26
-; GFX7-NEXT: v_readfirstlane_b32 s35, v0
-; GFX7-NEXT: v_mul_hi_u32 v0, v5, s10
+; GFX7-NEXT: v_readfirstlane_b32 s35, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s4, v3
; GFX7-NEXT: s_mul_i32 s28, s3, s11
; GFX7-NEXT: s_add_u32 s25, s28, s25
; GFX7-NEXT: s_addc_u32 s26, s35, s26
-; GFX7-NEXT: v_readfirstlane_b32 s35, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mul_hi_u32 v6, v0, s9
+; GFX7-NEXT: v_readfirstlane_b32 s35, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s5, v4
; GFX7-NEXT: s_mul_i32 s28, s4, s10
; GFX7-NEXT: s_add_u32 s25, s28, s25
-; GFX7-NEXT: v_mul_hi_u32 v1, s1, v1
; GFX7-NEXT: s_addc_u32 s26, s35, s26
-; GFX7-NEXT: v_readfirstlane_b32 s35, v6
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mul_hi_u32 v6, v6, s8
+; GFX7-NEXT: v_readfirstlane_b32 s35, v5
+; GFX7-NEXT: v_mul_hi_u32 v5, s6, v1
; GFX7-NEXT: s_mul_i32 s28, s5, s9
+; GFX7-NEXT: v_mul_hi_u32 v0, v0, s13
; GFX7-NEXT: s_add_u32 s25, s28, s25
-; GFX7-NEXT: v_mul_hi_u32 v2, s16, v2
-; GFX7-NEXT: v_readfirstlane_b32 s36, v1
-; GFX7-NEXT: v_mul_hi_u32 v1, s2, v4
; GFX7-NEXT: s_addc_u32 s26, s35, s26
; GFX7-NEXT: s_mul_i32 s28, s6, s8
-; GFX7-NEXT: v_readfirstlane_b32 s35, v6
+; GFX7-NEXT: v_readfirstlane_b32 s35, v5
; GFX7-NEXT: s_add_u32 s25, s28, s25
; GFX7-NEXT: s_addc_u32 s26, s35, s26
+; GFX7-NEXT: v_readfirstlane_b32 s35, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, v2, s12
; GFX7-NEXT: s_mul_i32 s28, s16, s13
-; GFX7-NEXT: v_readfirstlane_b32 s35, v2
; GFX7-NEXT: s_add_u32 s27, s28, s27
-; GFX7-NEXT: v_readfirstlane_b32 s37, v1
-; GFX7-NEXT: v_mul_hi_u32 v1, v3, s10
; GFX7-NEXT: s_addc_u32 s25, s35, s25
+; GFX7-NEXT: v_readfirstlane_b32 s36, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, v6, s11
; GFX7-NEXT: s_mul_i32 s35, s1, s12
; GFX7-NEXT: s_cselect_b32 s28, 1, 0
; GFX7-NEXT: s_add_u32 s27, s35, s27
+; GFX7-NEXT: v_readfirstlane_b32 s37, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, s3, v3
; GFX7-NEXT: s_addc_u32 s25, s36, s25
; GFX7-NEXT: s_mul_i32 s36, s2, s11
; GFX7-NEXT: s_cselect_b32 s35, 1, 0
+; GFX7-NEXT: v_readfirstlane_b32 s38, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, s4, v4
; GFX7-NEXT: s_add_u32 s27, s36, s27
-; GFX7-NEXT: v_readfirstlane_b32 s38, v1
-; GFX7-NEXT: v_mul_hi_u32 v1, v5, s9
; GFX7-NEXT: s_addc_u32 s25, s37, s25
; GFX7-NEXT: s_mul_i32 s37, s3, s10
; GFX7-NEXT: s_cselect_b32 s36, 1, 0
; GFX7-NEXT: s_add_u32 s27, s37, s27
-; GFX7-NEXT: v_mul_hi_u32 v0, v0, s8
+; GFX7-NEXT: v_readfirstlane_b32 s39, v0
+; GFX7-NEXT: v_mul_hi_u32 v0, s5, v1
; GFX7-NEXT: s_addc_u32 s25, s38, s25
; GFX7-NEXT: s_mul_i32 s38, s4, s9
; GFX7-NEXT: s_cselect_b32 s37, 1, 0
-; GFX7-NEXT: v_readfirstlane_b32 s39, v1
; GFX7-NEXT: s_add_u32 s27, s38, s27
; GFX7-NEXT: s_addc_u32 s25, s39, s25
; GFX7-NEXT: s_mul_i32 s39, s5, s8
@@ -1965,8 +1953,8 @@ define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
; GFX7-NEXT: s_mul_i32 s7, s7, s8
; GFX7-NEXT: s_mul_i32 s0, s0, s8
; GFX7-NEXT: s_add_u32 s7, s7, s1
-; GFX7-NEXT: s_mov_b32 s1, s18
-; GFX7-NEXT: s_mov_b32 s2, s17
+; GFX7-NEXT: s_mov_b32 s1, s17
+; GFX7-NEXT: s_mov_b32 s2, s18
; GFX7-NEXT: s_mov_b32 s3, s19
; GFX7-NEXT: s_mov_b32 s4, s20
; GFX7-NEXT: s_mov_b32 s5, s21
@@ -1975,85 +1963,81 @@ define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
;
; GFX8-LABEL: s_mul_i256:
; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v4, s9
+; GFX8-NEXT: v_mul_hi_u32 v5, s1, v4
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
+; GFX8-NEXT: v_mul_hi_u32 v2, v0, v1
+; GFX8-NEXT: v_readfirstlane_b32 s21, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s2, v1
; GFX8-NEXT: s_mov_b32 s16, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: v_mul_hi_u32 v2, s1, v1
-; GFX8-NEXT: v_mul_hi_u32 v1, s16, v1
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
-; GFX8-NEXT: v_readfirstlane_b32 s21, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mul_hi_u32 v3, v2, s8
+; GFX8-NEXT: v_readfirstlane_b32 s17, v2
+; GFX8-NEXT: v_mul_hi_u32 v2, v0, s10
+; GFX8-NEXT: v_readfirstlane_b32 s22, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v0, v4
; GFX8-NEXT: s_mul_i32 s18, s16, s10
; GFX8-NEXT: s_mul_i32 s20, s1, s9
-; GFX8-NEXT: v_readfirstlane_b32 s19, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_readfirstlane_b32 s19, v2
; GFX8-NEXT: s_add_u32 s18, s20, s18
; GFX8-NEXT: s_addc_u32 s19, s21, s19
; GFX8-NEXT: s_mul_i32 s21, s2, s8
-; GFX8-NEXT: v_readfirstlane_b32 s23, v1
-; GFX8-NEXT: v_mul_hi_u32 v1, v0, s8
+; GFX8-NEXT: v_readfirstlane_b32 s23, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s1, v1
; GFX8-NEXT: s_cselect_b32 s20, 1, 0
-; GFX8-NEXT: v_readfirstlane_b32 s22, v3
; GFX8-NEXT: s_add_u32 s18, s21, s18
; GFX8-NEXT: s_addc_u32 s19, s22, s19
; GFX8-NEXT: s_mul_i32 s22, s16, s9
; GFX8-NEXT: s_cselect_b32 s21, 1, 0
; GFX8-NEXT: s_add_u32 s17, s22, s17
-; GFX8-NEXT: s_addc_u32 s22, s23, s18
-; GFX8-NEXT: v_readfirstlane_b32 s23, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: v_mul_hi_u32 v3, s16, v1
-; GFX8-NEXT: s_mul_i32 s18, s1, s8
+; GFX8-NEXT: s_addc_u32 s18, s23, s18
+; GFX8-NEXT: v_readfirstlane_b32 s23, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v0, s12
+; GFX8-NEXT: s_mul_i32 s22, s1, s8
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
; GFX8-NEXT: s_cselect_b32 s25, 1, 0
-; GFX8-NEXT: s_add_u32 s18, s18, s17
-; GFX8-NEXT: s_addc_u32 s17, s23, s22
-; GFX8-NEXT: v_mov_b32_e32 v4, s11
-; GFX8-NEXT: v_readfirstlane_b32 s23, v3
-; GFX8-NEXT: v_mul_hi_u32 v3, v2, s10
-; GFX8-NEXT: v_mul_hi_u32 v5, s1, v4
+; GFX8-NEXT: s_add_u32 s17, s22, s17
+; GFX8-NEXT: s_addc_u32 s18, s23, s18
+; GFX8-NEXT: v_readfirstlane_b32 s23, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v6, s10
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_mul_hi_u32 v7, v2, s11
; GFX8-NEXT: s_mul_i32 s22, s16, s12
+; GFX8-NEXT: v_readfirstlane_b32 s28, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s3, v4
; GFX8-NEXT: s_mul_i32 s24, s1, s11
-; GFX8-NEXT: v_readfirstlane_b32 s28, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_readfirstlane_b32 s27, v5
-; GFX8-NEXT: v_mul_hi_u32 v5, v3, s9
; GFX8-NEXT: s_cselect_b32 s26, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s27, v7
+; GFX8-NEXT: v_readfirstlane_b32 s29, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s4, v1
; GFX8-NEXT: s_add_u32 s24, s24, s22
; GFX8-NEXT: s_addc_u32 s23, s27, s23
-; GFX8-NEXT: v_readfirstlane_b32 s29, v5
-; GFX8-NEXT: v_mov_b32_e32 v5, s4
-; GFX8-NEXT: v_mul_hi_u32 v6, v5, s8
; GFX8-NEXT: s_mul_i32 s27, s2, s10
+; GFX8-NEXT: v_readfirstlane_b32 s30, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v0, s11
; GFX8-NEXT: s_cselect_b32 s22, 1, 0
; GFX8-NEXT: s_add_u32 s24, s27, s24
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s10
; GFX8-NEXT: s_addc_u32 s27, s28, s23
+; GFX8-NEXT: v_readfirstlane_b32 s31, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v2, s10
; GFX8-NEXT: s_mul_i32 s28, s3, s9
; GFX8-NEXT: s_cselect_b32 s23, 1, 0
; GFX8-NEXT: s_add_u32 s28, s28, s24
-; GFX8-NEXT: v_readfirstlane_b32 s30, v6
-; GFX8-NEXT: v_mul_hi_u32 v6, s16, v4
; GFX8-NEXT: s_addc_u32 s27, s29, s27
; GFX8-NEXT: s_mul_i32 s29, s4, s8
; GFX8-NEXT: s_cselect_b32 s24, 1, 0
; GFX8-NEXT: s_add_u32 s28, s29, s28
-; GFX8-NEXT: v_readfirstlane_b32 s33, v0
-; GFX8-NEXT: v_mul_hi_u32 v0, v2, s9
+; GFX8-NEXT: v_readfirstlane_b32 s33, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s2, v4
; GFX8-NEXT: s_addc_u32 s27, s30, s27
; GFX8-NEXT: s_mul_i32 s30, s16, s11
; GFX8-NEXT: s_cselect_b32 s29, 1, 0
-; GFX8-NEXT: v_readfirstlane_b32 s31, v6
; GFX8-NEXT: s_add_u32 s19, s30, s19
; GFX8-NEXT: s_addc_u32 s28, s31, s28
; GFX8-NEXT: s_mul_i32 s31, s1, s10
; GFX8-NEXT: s_cselect_b32 s30, 1, 0
; GFX8-NEXT: s_add_u32 s19, s31, s19
-; GFX8-NEXT: v_readfirstlane_b32 s34, v0
-; GFX8-NEXT: v_mul_hi_u32 v0, v3, s8
+; GFX8-NEXT: v_readfirstlane_b32 s34, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s3, v1
; GFX8-NEXT: s_addc_u32 s28, s33, s28
; GFX8-NEXT: s_mul_i32 s33, s2, s9
; GFX8-NEXT: s_cselect_b32 s31, 1, 0
@@ -2061,87 +2045,85 @@ define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
; GFX8-NEXT: s_addc_u32 s28, s34, s28
; GFX8-NEXT: s_mul_i32 s34, s3, s8
; GFX8-NEXT: s_cselect_b32 s33, 1, 0
-; GFX8-NEXT: v_readfirstlane_b32 s35, v0
+; GFX8-NEXT: v_readfirstlane_b32 s35, v5
; GFX8-NEXT: s_add_u32 s19, s34, s19
; GFX8-NEXT: s_addc_u32 s28, s35, s28
; GFX8-NEXT: s_cselect_b32 s34, 1, 0
; GFX8-NEXT: s_cmp_lg_u32 s25, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s14
; GFX8-NEXT: s_cselect_b32 s25, 1, 0
; GFX8-NEXT: s_cmp_lg_u32 s26, 0
-; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX8-NEXT: v_mul_hi_u32 v5, v0, s14
; GFX8-NEXT: s_addc_u32 s19, s25, s19
; GFX8-NEXT: s_cselect_b32 s25, 1, 0
; GFX8-NEXT: s_cmp_lg_u32 s20, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s13
; GFX8-NEXT: s_cselect_b32 s20, 1, 0
; GFX8-NEXT: s_cmp_lg_u32 s21, 0
-; GFX8-NEXT: v_mul_hi_u32 v6, s1, v2
+; GFX8-NEXT: v_mul_hi_u32 v8, v2, s13
; GFX8-NEXT: s_addc_u32 s20, s20, 0
-; GFX8-NEXT: v_readfirstlane_b32 s26, v0
-; GFX8-NEXT: v_mul_hi_u32 v0, s2, v1
+; GFX8-NEXT: v_readfirstlane_b32 s26, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v6, s12
; GFX8-NEXT: s_cmp_lg_u32 s25, 0
; GFX8-NEXT: s_addc_u32 s20, s20, s28
; GFX8-NEXT: s_mul_i32 s25, s16, s14
; GFX8-NEXT: s_mul_i32 s28, s1, s13
+; GFX8-NEXT: v_mov_b32_e32 v7, s3
; GFX8-NEXT: s_cselect_b32 s21, 1, 0
-; GFX8-NEXT: v_readfirstlane_b32 s35, v6
+; GFX8-NEXT: v_readfirstlane_b32 s35, v8
; GFX8-NEXT: s_add_u32 s25, s28, s25
; GFX8-NEXT: s_addc_u32 s26, s35, s26
-; GFX8-NEXT: v_readfirstlane_b32 s35, v0
-; GFX8-NEXT: v_mul_hi_u32 v0, v3, s11
+; GFX8-NEXT: v_readfirstlane_b32 s35, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, v7, s11
; GFX8-NEXT: s_mul_i32 s28, s2, s12
+; GFX8-NEXT: v_mov_b32_e32 v3, s10
; GFX8-NEXT: s_add_u32 s25, s28, s25
; GFX8-NEXT: s_addc_u32 s26, s35, s26
-; GFX8-NEXT: v_readfirstlane_b32 s35, v0
-; GFX8-NEXT: v_mul_hi_u32 v0, v5, s10
+; GFX8-NEXT: v_readfirstlane_b32 s35, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s4, v3
; GFX8-NEXT: s_mul_i32 s28, s3, s11
; GFX8-NEXT: s_add_u32 s25, s28, s25
; GFX8-NEXT: s_addc_u32 s26, s35, s26
-; GFX8-NEXT: v_readfirstlane_b32 s35, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mul_hi_u32 v6, v0, s9
+; GFX8-NEXT: v_readfirstlane_b32 s35, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s5, v4
; GFX8-NEXT: s_mul_i32 s28, s4, s10
; GFX8-NEXT: s_add_u32 s25, s28, s25
-; GFX8-NEXT: v_mul_hi_u32 v1, s1, v1
; GFX8-NEXT: s_addc_u32 s26, s35, s26
-; GFX8-NEXT: v_readfirstlane_b32 s35, v6
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mul_hi_u32 v6, v6, s8
+; GFX8-NEXT: v_readfirstlane_b32 s35, v5
+; GFX8-NEXT: v_mul_hi_u32 v5, s6, v1
; GFX8-NEXT: s_mul_i32 s28, s5, s9
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s13
; GFX8-NEXT: s_add_u32 s25, s28, s25
-; GFX8-NEXT: v_mul_hi_u32 v2, s16, v2
-; GFX8-NEXT: v_readfirstlane_b32 s36, v1
-; GFX8-NEXT: v_mul_hi_u32 v1, s2, v4
; GFX8-NEXT: s_addc_u32 s26, s35, s26
; GFX8-NEXT: s_mul_i32 s28, s6, s8
-; GFX8-NEXT: v_readfirstlane_b32 s35, v6
+; GFX8-NEXT: v_readfirstlane_b32 s35, v5
; GFX8-NEXT: s_add_u32 s25, s28, s25
; GFX8-NEXT: s_addc_u32 s26, s35, s26
+; GFX8-NEXT: v_readfirstlane_b32 s35, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, v2, s12
; GFX8-NEXT: s_mul_i32 s28, s16, s13
-; GFX8-NEXT: v_readfirstlane_b32 s35, v2
; GFX8-NEXT: s_add_u32 s27, s28, s27
-; GFX8-NEXT: v_readfirstlane_b32 s37, v1
-; GFX8-NEXT: v_mul_hi_u32 v1, v3, s10
; GFX8-NEXT: s_addc_u32 s25, s35, s25
+; GFX8-NEXT: v_readfirstlane_b32 s36, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, v6, s11
; GFX8-NEXT: s_mul_i32 s35, s1, s12
; GFX8-NEXT: s_cselect_b32 s28, 1, 0
; GFX8-NEXT: s_add_u32 s27, s35, s27
+; GFX8-NEXT: v_readfirstlane_b32 s37, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, s3, v3
; GFX8-NEXT: s_addc_u32 s25, s36, s25
; GFX8-NEXT: s_mul_i32 s36, s2, s11
; GFX8-NEXT: s_cselect_b32 s35, 1, 0
+; GFX8-NEXT: v_readfirstlane_b32 s38, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, s4, v4
; GFX8-NEXT: s_add_u32 s27, s36, s27
-; GFX8-NEXT: v_readfirstlane_b32 s38, v1
-; GFX8-NEXT: v_mul_hi_u32 v1, v5, s9
; GFX8-NEXT: s_addc_u32 s25, s37, s25
; GFX8-NEXT: s_mul_i32 s37, s3, s10
; GFX8-NEXT: s_cselect_b32 s36, 1, 0
; GFX8-NEXT: s_add_u32 s27, s37, s27
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s8
+; GFX8-NEXT: v_readfirstlane_b32 s39, v0
+; GFX8-NEXT: v_mul_hi_u32 v0, s5, v1
; GFX8-NEXT: s_addc_u32 s25, s38, s25
; GFX8-NEXT: s_mul_i32 s38, s4, s9
; GFX8-NEXT: s_cselect_b32 s37, 1, 0
-; GFX8-NEXT: v_readfirstlane_b32 s39, v1
; GFX8-NEXT: s_add_u32 s27, s38, s27
; GFX8-NEXT: s_addc_u32 s25, s39, s25
; GFX8-NEXT: s_mul_i32 s39, s5, s8
@@ -2194,8 +2176,8 @@ define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
; GFX8-NEXT: s_mul_i32 s7, s7, s8
; GFX8-NEXT: s_mul_i32 s0, s0, s8
; GFX8-NEXT: s_add_u32 s7, s7, s1
-; GFX8-NEXT: s_mov_b32 s1, s18
-; GFX8-NEXT: s_mov_b32 s2, s17
+; GFX8-NEXT: s_mov_b32 s1, s17
+; GFX8-NEXT: s_mov_b32 s2, s18
; GFX8-NEXT: s_mov_b32 s3, s19
; GFX8-NEXT: s_mov_b32 s4, s20
; GFX8-NEXT: s_mov_b32 s5, s21
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
index c226f7f9289ee..1138f5bf9bb96 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
@@ -291,28 +291,16 @@ define amdgpu_ps <2 x float> @v_orn2_i64_vs(i64 %src0, i64 inreg %src1) {
; GCN-LABEL: v_orn2_i64_vs:
; GCN: ; %bb.0:
; GCN-NEXT: s_not_b64 s[0:1], s[2:3]
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_or_b32_e32 v0, v0, v2
-; GCN-NEXT: v_or_b32_e32 v1, v1, v3
+; GCN-NEXT: v_or_b32_e32 v0, s0, v0
+; GCN-NEXT: v_or_b32_e32 v1, s1, v1
; GCN-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: v_orn2_i64_vs:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: v_orn2_i64_vs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_not_b64 s[0:1], s[2:3]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: ; return to shader part epilog
+; GFX10PLUS-LABEL: v_orn2_i64_vs:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_not_b64 s[0:1], s[2:3]
+; GFX10PLUS-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX10PLUS-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX10PLUS-NEXT: ; return to shader part epilog
%not.src1 = xor i64 %src1, -1
%or = or i64 %src0, %not.src1
%cast = bitcast i64 %or to <2 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 7375704ae8c42..c0aa49548075f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -257,29 +257,29 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_i32 s15, s15, s16
; CHECK-NEXT: s_add_u32 s13, s13, s14
; CHECK-NEXT: v_mov_b32_e32 v0, s13
-; CHECK-NEXT: v_mul_hi_u32 v0, s3, v0
+; CHECK-NEXT: v_mul_hi_u32 v1, s3, v0
; CHECK-NEXT: s_mul_i32 s14, s3, s13
; CHECK-NEXT: s_addc_u32 s12, s12, s15
; CHECK-NEXT: s_mul_i32 s3, s3, s12
-; CHECK-NEXT: v_readfirstlane_b32 s15, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s14
-; CHECK-NEXT: v_mul_hi_u32 v1, s13, v0
+; CHECK-NEXT: v_readfirstlane_b32 s15, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, v0, s14
; CHECK-NEXT: s_add_u32 s3, s3, s15
; CHECK-NEXT: s_mul_i32 s5, s5, s13
; CHECK-NEXT: s_add_u32 s3, s5, s3
+; CHECK-NEXT: v_mov_b32_e32 v2, s14
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: s_mul_i32 s15, s13, s3
-; CHECK-NEXT: v_mul_hi_u32 v0, s12, v0
-; CHECK-NEXT: v_mul_hi_u32 v2, s13, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, s12, v2
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
; CHECK-NEXT: s_mul_i32 s14, s12, s14
; CHECK-NEXT: s_add_u32 s5, s5, s15
; CHECK-NEXT: s_cselect_b32 s15, 1, 0
; CHECK-NEXT: s_add_u32 s5, s5, s14
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s15, s15, s5
-; CHECK-NEXT: v_readfirstlane_b32 s5, v0
-; CHECK-NEXT: v_readfirstlane_b32 s14, v2
+; CHECK-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-NEXT: v_readfirstlane_b32 s14, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: s_mul_i32 s3, s12, s3
; CHECK-NEXT: s_add_u32 s5, s5, s14
; CHECK-NEXT: s_cselect_b32 s14, 1, 0
@@ -296,12 +296,12 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_mov_b32_e32 v0, s3
; CHECK-NEXT: v_mul_hi_u32 v0, s10, v0
; CHECK-NEXT: s_addc_u32 s5, s12, s14
-; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mov_b32_e32 v2, s5
; CHECK-NEXT: s_mul_i32 s13, s10, s5
; CHECK-NEXT: v_readfirstlane_b32 s12, v0
; CHECK-NEXT: v_mov_b32_e32 v0, s11
; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
-; CHECK-NEXT: v_mul_hi_u32 v2, s10, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, s10, v2
; CHECK-NEXT: s_mul_i32 s14, s11, s3
; CHECK-NEXT: s_add_u32 s12, s12, s13
; CHECK-NEXT: s_cselect_b32 s13, 1, 0
@@ -309,17 +309,17 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_cselect_b32 s3, 1, 0
; CHECK-NEXT: s_add_i32 s13, s13, s3
; CHECK-NEXT: v_readfirstlane_b32 s3, v0
-; CHECK-NEXT: v_readfirstlane_b32 s12, v2
+; CHECK-NEXT: v_readfirstlane_b32 s12, v3
; CHECK-NEXT: s_mul_i32 s5, s11, s5
; CHECK-NEXT: s_add_u32 s3, s3, s12
; CHECK-NEXT: s_cselect_b32 s12, 1, 0
; CHECK-NEXT: s_add_u32 s3, s3, s5
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s5, s12, s5
+; CHECK-NEXT: v_mul_hi_u32 v0, s11, v2
+; CHECK-NEXT: v_mov_b32_e32 v1, s8
; CHECK-NEXT: s_add_u32 s12, s3, s13
-; CHECK-NEXT: v_mul_hi_u32 v0, s11, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s12
-; CHECK-NEXT: v_mul_hi_u32 v1, s8, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, v1, s12
; CHECK-NEXT: s_cselect_b32 s3, 1, 0
; CHECK-NEXT: s_add_i32 s5, s5, s3
; CHECK-NEXT: v_readfirstlane_b32 s3, v0
@@ -1089,24 +1089,24 @@ define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_cvt_f32_u32_e32 v3, 0x12d8fb
; CHECK-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, 0xffed2705
; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT: v_xor_b32_e32 v5, v0, v2
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; CHECK-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; CHECK-NEXT: v_trunc_f32_e32 v4, v4
; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v4
-; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
-; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
-; CHECK-NEXT: v_mul_hi_u32 v6, v4, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v4, v4
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
+; CHECK-NEXT: v_mul_hi_u32 v6, v5, v3
; CHECK-NEXT: v_readfirstlane_b32 s4, v3
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CHECK-NEXT: v_readfirstlane_b32 s5, v0
+; CHECK-NEXT: v_readfirstlane_b32 s5, v4
; CHECK-NEXT: v_readfirstlane_b32 s7, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v3, s6
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
@@ -1115,7 +1115,7 @@ define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: v_readfirstlane_b32 s8, v6
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v6, v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, s6
; CHECK-NEXT: v_mul_hi_u32 v3, v3, s7
; CHECK-NEXT: s_mul_i32 s10, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
@@ -1130,99 +1130,100 @@ define i64 @v_sdiv_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_u32 s6, s6, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v0, v0, s7
+; CHECK-NEXT: v_mul_hi_u32 v3, v4, s7
; CHECK-NEXT: s_add_i32 s8, s8, s10
; CHECK-NEXT: s_add_u32 s6, s6, s9
; CHECK-NEXT: s_cselect_b32 s7, 1, 0
; CHECK-NEXT: s_add_i32 s8, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v3
; CHECK-NEXT: s_add_i32 s7, s7, s8
; CHECK-NEXT: s_add_u32 s4, s4, s6
-; CHECK-NEXT: v_mul_hi_u32 v0, v4, s4
+; CHECK-NEXT: v_mul_hi_u32 v3, v5, s4
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CHECK-NEXT: v_mov_b32_e32 v4, s4
; CHECK-NEXT: s_addc_u32 s5, s5, s7
-; CHECK-NEXT: v_xor_b32_e32 v6, v1, v2
-; CHECK-NEXT: v_readfirstlane_b32 s7, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s6
-; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v3
+; CHECK-NEXT: v_mul_hi_u32 v3, v4, s6
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: s_mul_i32 s8, s4, -1
; CHECK-NEXT: s_add_u32 s7, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s8, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s7
+; CHECK-NEXT: v_mov_b32_e32 v5, s6
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v0, s5, v0
-; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, s5, v5
+; CHECK-NEXT: v_mul_hi_u32 v4, v4, s7
; CHECK-NEXT: s_mul_i32 s6, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_u32 s6, s8, s6
; CHECK-NEXT: s_cselect_b32 s6, 1, 0
; CHECK-NEXT: s_add_i32 s9, s9, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v0
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: v_readfirstlane_b32 s6, v3
+; CHECK-NEXT: v_readfirstlane_b32 s8, v4
+; CHECK-NEXT: v_mov_b32_e32 v3, s7
; CHECK-NEXT: s_mul_i32 s7, s5, s7
; CHECK-NEXT: s_add_u32 s6, s6, s8
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_u32 s6, s6, s7
; CHECK-NEXT: s_cselect_b32 s7, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v0, s5, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, s5, v3
; CHECK-NEXT: s_add_i32 s7, s8, s7
; CHECK-NEXT: s_add_u32 s6, s6, s9
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_i32 s7, s7, s8
-; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
; CHECK-NEXT: s_add_i32 s8, s8, s7
; CHECK-NEXT: s_add_u32 s4, s4, s6
; CHECK-NEXT: s_addc_u32 s5, s5, s8
-; CHECK-NEXT: v_mul_hi_u32 v0, v5, s4
-; CHECK-NEXT: v_mul_lo_u32 v1, v5, s5
-; CHECK-NEXT: v_mul_lo_u32 v3, v6, s4
-; CHECK-NEXT: v_mul_hi_u32 v4, v5, s5
-; CHECK-NEXT: v_mul_hi_u32 v9, v6, s5
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v3, v6, s4
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; CHECK-NEXT: v_mul_lo_u32 v1, v6, s5
+; CHECK-NEXT: v_mul_hi_u32 v3, v0, s4
+; CHECK-NEXT: v_mul_lo_u32 v4, v0, s5
+; CHECK-NEXT: v_mul_lo_u32 v5, v1, s4
+; CHECK-NEXT: v_mul_hi_u32 v6, v0, s5
+; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v5
; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v1, s4
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v1, v0
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v8, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v3
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v9, v[1:2]
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v5, v0
-; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], v6, v3, vcc
-; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
-; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_mul_lo_u32 v4, v1, s5
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v4, v3
+; CHECK-NEXT: v_mul_hi_u32 v9, v1, s5
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v8, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, v9, v5
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v9, v[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_subb_u32_e64 v3, s[4:5], v1, v5, vcc
+; CHECK-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v5
+; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_cndmask_b32_e64 v1, -1, v4, s[4:5]
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cndmask_b32_e64 v3, -1, v4, s[4:5]
; CHECK-NEXT: v_add_i32_e32 v4, vcc, 1, v8
; CHECK-NEXT: v_addc_u32_e32 v5, vcc, 0, v9, vcc
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v7
; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; CHECK-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 1, v4
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, 1, v4
; CHECK-NEXT: v_addc_u32_e32 v6, vcc, 0, v5, vcc
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
; CHECK-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
@@ -1236,190 +1237,191 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-LABEL: v_sdiv_v2i64_oddk_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0
-; GISEL-NEXT: s_mov_b32 s8, 1
-; GISEL-NEXT: s_cmp_lg_u32 s8, 0
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v5, 0x12d8fb
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v6, 0
; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
-; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
-; GISEL-NEXT: v_trunc_f32_e32 v6, v6
-; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
-; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v5
-; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v1, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v0, v4
+; GISEL-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, v1, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v8, v0, v4
+; GISEL-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v5
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x2f800000, v1
+; GISEL-NEXT: v_trunc_f32_e32 v5, v5
+; GISEL-NEXT: v_mac_f32_e32 v1, 0xcf800000, v5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v6, v1
; GISEL-NEXT: v_mov_b32_e32 v0, 0xffed2705
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, v7
-; GISEL-NEXT: v_readfirstlane_b32 s7, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v1, v5
+; GISEL-NEXT: s_mov_b32 s8, 1
+; GISEL-NEXT: v_mul_hi_u32 v5, v0, v6
+; GISEL-NEXT: v_readfirstlane_b32 s7, v6
; GISEL-NEXT: s_mul_i32 s5, s7, 0xffed2705
-; GISEL-NEXT: v_readfirstlane_b32 s6, v5
-; GISEL-NEXT: v_readfirstlane_b32 s9, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v7, s5
+; GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v5
+; GISEL-NEXT: v_mul_hi_u32 v5, v6, s5
; GISEL-NEXT: s_subb_u32 s4, 0, 0
; GISEL-NEXT: s_mul_i32 s10, s6, 0xffed2705
; GISEL-NEXT: s_add_u32 s9, s10, s9
; GISEL-NEXT: s_mul_i32 s10, s4, s7
; GISEL-NEXT: s_add_u32 s10, s10, s9
-; GISEL-NEXT: v_readfirstlane_b32 s11, v8
+; GISEL-NEXT: v_readfirstlane_b32 s11, v5
; GISEL-NEXT: s_mul_i32 s12, s7, s10
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s5
-; GISEL-NEXT: v_mul_hi_u32 v9, v7, s10
+; GISEL-NEXT: v_mul_hi_u32 v5, v1, s5
+; GISEL-NEXT: v_mul_hi_u32 v9, v6, s10
; GISEL-NEXT: s_mul_i32 s13, s6, s5
; GISEL-NEXT: s_add_u32 s12, s11, s12
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-NEXT: s_add_u32 s12, s12, s13
; GISEL-NEXT: s_cselect_b32 s5, 1, 0
; GISEL-NEXT: s_add_i32 s14, s14, s5
-; GISEL-NEXT: v_readfirstlane_b32 s12, v8
+; GISEL-NEXT: v_readfirstlane_b32 s12, v5
; GISEL-NEXT: v_readfirstlane_b32 s5, v9
; GISEL-NEXT: s_mul_i32 s15, s6, s10
; GISEL-NEXT: s_add_u32 s5, s12, s5
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-NEXT: s_add_u32 s5, s5, s15
; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s10
+; GISEL-NEXT: v_mul_hi_u32 v5, v1, s10
; GISEL-NEXT: s_add_i32 s15, s16, s15
; GISEL-NEXT: s_add_u32 s5, s5, s14
; GISEL-NEXT: s_cselect_b32 s10, 1, 0
; GISEL-NEXT: s_add_i32 s15, s15, s10
-; GISEL-NEXT: v_readfirstlane_b32 s10, v8
+; GISEL-NEXT: v_readfirstlane_b32 s10, v5
; GISEL-NEXT: s_add_i32 s10, s10, s15
; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, s5
+; GISEL-NEXT: v_mul_hi_u32 v5, v0, s5
; GISEL-NEXT: s_mul_i32 s14, s5, 0xffed2705
-; GISEL-NEXT: v_xor_b32_e32 v14, v6, v4
-; GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GISEL-NEXT: v_mov_b32_e32 v9, s5
; GISEL-NEXT: s_addc_u32 s10, s6, s10
-; GISEL-NEXT: v_readfirstlane_b32 s15, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, s5, v6
+; GISEL-NEXT: v_readfirstlane_b32 s15, v5
+; GISEL-NEXT: v_mul_hi_u32 v5, v9, s14
; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
; GISEL-NEXT: s_add_u32 s15, s16, s15
; GISEL-NEXT: s_mul_i32 s4, s4, s5
; GISEL-NEXT: s_add_u32 s4, s4, s15
-; GISEL-NEXT: v_readfirstlane_b32 s15, v8
-; GISEL-NEXT: v_mov_b32_e32 v8, s4
+; GISEL-NEXT: v_mov_b32_e32 v10, s14
+; GISEL-NEXT: v_readfirstlane_b32 s15, v5
; GISEL-NEXT: s_mul_i32 s16, s5, s4
-; GISEL-NEXT: v_mul_hi_u32 v6, s10, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, s5, v8
+; GISEL-NEXT: v_mul_hi_u32 v5, s10, v10
+; GISEL-NEXT: v_mul_hi_u32 v9, v9, s4
; GISEL-NEXT: s_mul_i32 s14, s10, s14
; GISEL-NEXT: s_add_u32 s15, s15, s16
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-NEXT: s_add_u32 s14, s15, s14
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-NEXT: s_add_i32 s16, s16, s14
-; GISEL-NEXT: v_readfirstlane_b32 s14, v6
+; GISEL-NEXT: v_readfirstlane_b32 s14, v5
; GISEL-NEXT: v_readfirstlane_b32 s15, v9
+; GISEL-NEXT: v_mov_b32_e32 v5, s4
; GISEL-NEXT: s_mul_i32 s4, s10, s4
; GISEL-NEXT: s_add_u32 s14, s14, s15
; GISEL-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-NEXT: s_add_u32 s4, s14, s4
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v6, s10, v8
+; GISEL-NEXT: v_mul_hi_u32 v5, s10, v5
; GISEL-NEXT: s_add_i32 s14, s15, s14
; GISEL-NEXT: s_add_u32 s4, s4, s16
; GISEL-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-NEXT: s_add_i32 s14, s14, s15
-; GISEL-NEXT: v_readfirstlane_b32 s15, v6
+; GISEL-NEXT: v_readfirstlane_b32 s15, v5
; GISEL-NEXT: s_add_i32 s15, s15, s14
; GISEL-NEXT: s_add_u32 s4, s5, s4
; GISEL-NEXT: s_addc_u32 s5, s10, s15
-; GISEL-NEXT: v_mul_hi_u32 v8, v1, s4
-; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, s4
-; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
-; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_xor_b32_e32 v7, v7, v4
+; GISEL-NEXT: v_mul_hi_u32 v9, v8, s4
+; GISEL-NEXT: v_mul_lo_u32 v10, v8, s5
+; GISEL-NEXT: v_mul_lo_u32 v11, v7, s4
+; GISEL-NEXT: v_mul_hi_u32 v12, v8, s5
+; GISEL-NEXT: v_mov_b32_e32 v5, 0x12d8fb
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v10, v14, s4
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v7, s4
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v10, v7, s5
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v8
-; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v15, 0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, v12, v10
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v16, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v10, v9
+; GISEL-NEXT: v_mul_hi_u32 v13, v7, s5
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v15, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, v13, v11
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v5, v16, v[10:11]
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: s_cmp_lg_u32 s8, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], 0, v15, v[10:11]
-; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v6
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], 0, v15, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v9, s[4:5], v7, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], v7, v13
+; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v5
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v5
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, -1, v10, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v9
+; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v9, -1, v10, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v15
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v16, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v8, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v7, -1, v8, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, 1, v10
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v11, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v10, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v11, v12, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc
; GISEL-NEXT: s_subb_u32 s4, 0, 0
+; GISEL-NEXT: v_xor_b32_e32 v10, v7, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v3
; GISEL-NEXT: s_mul_i32 s5, s4, s7
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; GISEL-NEXT: s_add_u32 s5, s5, s9
+; GISEL-NEXT: v_xor_b32_e32 v12, v2, v7
; GISEL-NEXT: s_mul_i32 s8, s7, s5
-; GISEL-NEXT: v_mul_hi_u32 v7, v7, s5
+; GISEL-NEXT: v_mul_hi_u32 v2, v6, s5
; GISEL-NEXT: s_add_u32 s8, s11, s8
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-NEXT: s_add_u32 s8, s8, s13
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-NEXT: s_add_i32 s9, s9, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; GISEL-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-NEXT: s_mul_i32 s10, s6, s5
; GISEL-NEXT: s_add_u32 s8, s12, s8
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v6
; GISEL-NEXT: s_cselect_b32 s11, 1, 0
; GISEL-NEXT: s_add_u32 s8, s8, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
; GISEL-NEXT: s_cselect_b32 s10, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v5, v5, s5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v15
+; GISEL-NEXT: v_mul_hi_u32 v1, v1, s5
; GISEL-NEXT: s_add_i32 s10, s11, s10
-; GISEL-NEXT: v_addc_u32_e32 v11, vcc, 0, v16, vcc
; GISEL-NEXT: s_add_u32 s5, s8, s9
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v1, v6
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
; GISEL-NEXT: s_add_i32 s10, s10, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v10
+; GISEL-NEXT: v_readfirstlane_b32 s8, v1
; GISEL-NEXT: s_add_i32 s8, s8, s10
-; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v11, vcc
; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GISEL-NEXT: v_mul_hi_u32 v0, v0, s5
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v10, v9, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v11, v12, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v16, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
; GISEL-NEXT: s_mul_i32 s7, s5, 0xffed2705
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v15, v1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
; GISEL-NEXT: s_addc_u32 s6, s6, s8
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GISEL-NEXT: v_mov_b32_e32 v0, s7
-; GISEL-NEXT: v_xor_b32_e32 v5, v2, v8
-; GISEL-NEXT: v_mul_hi_u32 v2, s5, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v1, s7
; GISEL-NEXT: s_mul_i32 s9, s6, 0xffed2705
; GISEL-NEXT: s_add_u32 s8, s9, s8
; GISEL-NEXT: s_mul_i32 s4, s4, s5
; GISEL-NEXT: s_add_u32 s4, s4, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v2
-; GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: s_mul_i32 s9, s5, s4
-; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
-; GISEL-NEXT: v_mul_hi_u32 v7, s5, v2
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
+; GISEL-NEXT: v_mul_hi_u32 v1, v1, s4
; GISEL-NEXT: s_mul_i32 s7, s6, s7
; GISEL-NEXT: s_add_u32 s8, s8, s9
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
@@ -1427,13 +1429,14 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: s_cselect_b32 s7, 1, 0
; GISEL-NEXT: s_add_i32 s9, s9, s7
; GISEL-NEXT: v_readfirstlane_b32 s7, v0
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
; GISEL-NEXT: s_mul_i32 s4, s6, s4
; GISEL-NEXT: s_add_u32 s7, s7, s8
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-NEXT: s_add_u32 s4, s7, s4
; GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
; GISEL-NEXT: s_add_i32 s7, s8, s7
; GISEL-NEXT: s_add_u32 s4, s4, s9
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
@@ -1441,64 +1444,64 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: s_add_i32 s8, s8, s7
; GISEL-NEXT: s_add_u32 s4, s5, s4
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
; GISEL-NEXT: s_addc_u32 s5, s6, s8
-; GISEL-NEXT: v_mul_hi_u32 v0, v5, s4
-; GISEL-NEXT: v_mul_lo_u32 v2, v5, s5
-; GISEL-NEXT: v_xor_b32_e32 v7, v3, v8
-; GISEL-NEXT: v_mul_lo_u32 v3, v7, s4
-; GISEL-NEXT: v_mul_hi_u32 v9, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v0, v12, s4
+; GISEL-NEXT: v_mul_lo_u32 v1, v12, s5
+; GISEL-NEXT: v_xor_b32_e32 v6, v3, v7
+; GISEL-NEXT: v_mul_lo_u32 v2, v6, s4
+; GISEL-NEXT: v_mul_hi_u32 v3, v12, s5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v7, s4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GISEL-NEXT: v_mul_lo_u32 v2, v7, s5
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_mul_hi_u32 v2, v6, s4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v6, s5
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v3
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v2, v0
-; GISEL-NEXT: v_mul_hi_u32 v10, v7, s5
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v2, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v3, v2
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v1, v0
+; GISEL-NEXT: v_mul_hi_u32 v1, v6, s5
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v13, 0
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v10, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v13, v[3:4]
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_add_i32_e32 v14, vcc, v1, v0
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v14, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v10, v4
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v12, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc
-; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v7, v3
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v13, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v6, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v5
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v5, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v12
-; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v13, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v6, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, 1, v13
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v14, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v5
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
; GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, 1, v5
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, 1, v6
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v8, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v8, v5, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v8
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v8
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v8
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v13, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v14, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_sdiv_v2i64_oddk_denom:
@@ -1510,29 +1513,29 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v5
; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v6
; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
-; CGP-NEXT: v_xor_b32_e32 v9, v0, v5
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
; CGP-NEXT: v_trunc_f32_e32 v6, v6
; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
; CGP-NEXT: v_cvt_u32_f32_e32 v7, v4
; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705
-; CGP-NEXT: v_cvt_u32_f32_e32 v0, v6
-; CGP-NEXT: v_xor_b32_e32 v10, v1, v5
-; CGP-NEXT: v_mul_hi_u32 v6, v4, v7
+; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT: v_mul_hi_u32 v8, v4, v7
; CGP-NEXT: v_readfirstlane_b32 s6, v7
; CGP-NEXT: s_mul_i32 s4, s6, 0xffed2705
-; CGP-NEXT: v_readfirstlane_b32 s7, v0
-; CGP-NEXT: v_readfirstlane_b32 s5, v6
-; CGP-NEXT: v_mul_hi_u32 v6, v7, s4
+; CGP-NEXT: v_readfirstlane_b32 s7, v6
+; CGP-NEXT: v_readfirstlane_b32 s5, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v7, s4
; CGP-NEXT: s_mul_i32 s8, s7, 0xffed2705
; CGP-NEXT: s_add_u32 s5, s8, s5
; CGP-NEXT: s_mul_i32 s8, s6, -1
; CGP-NEXT: s_add_u32 s5, s8, s5
-; CGP-NEXT: v_readfirstlane_b32 s8, v6
+; CGP-NEXT: v_readfirstlane_b32 s8, v8
; CGP-NEXT: s_mul_i32 s9, s6, s5
-; CGP-NEXT: v_mul_hi_u32 v6, v0, s4
+; CGP-NEXT: v_mul_hi_u32 v8, v6, s4
; CGP-NEXT: v_mul_hi_u32 v7, v7, s5
; CGP-NEXT: s_mul_i32 s10, s7, s4
; CGP-NEXT: s_add_u32 s11, s8, s9
@@ -1540,143 +1543,144 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: s_add_u32 s11, s11, s10
; CGP-NEXT: s_cselect_b32 s4, 1, 0
; CGP-NEXT: s_add_i32 s12, s12, s4
-; CGP-NEXT: v_readfirstlane_b32 s11, v6
+; CGP-NEXT: v_readfirstlane_b32 s11, v8
; CGP-NEXT: v_readfirstlane_b32 s13, v7
; CGP-NEXT: s_mul_i32 s14, s7, s5
; CGP-NEXT: s_add_u32 s4, s11, s13
; CGP-NEXT: s_cselect_b32 s15, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s14
; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v0, v0, s5
+; CGP-NEXT: v_mul_hi_u32 v6, v6, s5
; CGP-NEXT: s_add_i32 s15, s15, s16
; CGP-NEXT: s_add_u32 s4, s4, s12
; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: s_add_i32 s15, s15, s5
-; CGP-NEXT: v_readfirstlane_b32 s12, v0
+; CGP-NEXT: v_readfirstlane_b32 s12, v6
; CGP-NEXT: s_add_i32 s5, s12, s15
; CGP-NEXT: s_add_u32 s4, s6, s4
-; CGP-NEXT: v_mul_hi_u32 v0, v4, s4
+; CGP-NEXT: v_mul_hi_u32 v6, v4, s4
; CGP-NEXT: s_mul_i32 s15, s4, 0xffed2705
+; CGP-NEXT: v_mov_b32_e32 v7, s4
; CGP-NEXT: s_addc_u32 s5, s7, s5
+; CGP-NEXT: v_readfirstlane_b32 s16, v6
+; CGP-NEXT: v_mul_hi_u32 v6, v7, s15
; CGP-NEXT: s_mul_i32 s17, s5, 0xffed2705
-; CGP-NEXT: v_readfirstlane_b32 s16, v0
-; CGP-NEXT: v_mov_b32_e32 v0, s15
-; CGP-NEXT: v_mul_hi_u32 v1, s4, v0
; CGP-NEXT: s_add_u32 s16, s17, s16
; CGP-NEXT: s_mul_i32 s17, s4, -1
; CGP-NEXT: s_add_u32 s16, s17, s16
-; CGP-NEXT: v_readfirstlane_b32 s17, v1
-; CGP-NEXT: v_mov_b32_e32 v1, s16
+; CGP-NEXT: v_mov_b32_e32 v8, s15
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
; CGP-NEXT: s_mul_i32 s18, s4, s16
-; CGP-NEXT: v_mul_hi_u32 v0, s5, v0
-; CGP-NEXT: v_mul_hi_u32 v6, s4, v1
+; CGP-NEXT: v_mul_hi_u32 v6, s5, v8
+; CGP-NEXT: v_mul_hi_u32 v7, v7, s16
; CGP-NEXT: s_mul_i32 s15, s5, s15
; CGP-NEXT: s_add_u32 s17, s17, s18
; CGP-NEXT: s_cselect_b32 s18, 1, 0
; CGP-NEXT: s_add_u32 s15, s17, s15
; CGP-NEXT: s_cselect_b32 s15, 1, 0
; CGP-NEXT: s_add_i32 s18, s18, s15
-; CGP-NEXT: v_readfirstlane_b32 s15, v0
-; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: v_readfirstlane_b32 s15, v6
+; CGP-NEXT: v_readfirstlane_b32 s17, v7
+; CGP-NEXT: v_mov_b32_e32 v6, s16
; CGP-NEXT: s_mul_i32 s16, s5, s16
; CGP-NEXT: s_add_u32 s15, s15, s17
; CGP-NEXT: s_cselect_b32 s17, 1, 0
; CGP-NEXT: s_add_u32 s15, s15, s16
; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v0, s5, v1
+; CGP-NEXT: v_mul_hi_u32 v6, s5, v6
; CGP-NEXT: s_add_i32 s16, s17, s16
; CGP-NEXT: s_add_u32 s15, s15, s18
; CGP-NEXT: s_cselect_b32 s17, 1, 0
; CGP-NEXT: s_add_i32 s16, s16, s17
-; CGP-NEXT: v_readfirstlane_b32 s17, v0
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
; CGP-NEXT: s_add_i32 s17, s17, s16
; CGP-NEXT: s_add_u32 s4, s4, s15
; CGP-NEXT: s_addc_u32 s5, s5, s17
-; CGP-NEXT: v_mul_hi_u32 v0, v9, s4
-; CGP-NEXT: v_mul_lo_u32 v1, v9, s5
-; CGP-NEXT: v_mul_lo_u32 v7, v10, s4
-; CGP-NEXT: v_mul_hi_u32 v8, v9, s5
+; CGP-NEXT: v_mul_hi_u32 v7, v0, s4
+; CGP-NEXT: v_mul_lo_u32 v8, v0, s5
+; CGP-NEXT: v_mul_lo_u32 v9, v1, s4
+; CGP-NEXT: v_mul_hi_u32 v10, v0, s5
; CGP-NEXT: v_mov_b32_e32 v6, 0x12d8fb
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v7, v10, s4
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; CGP-NEXT: v_mul_lo_u32 v1, v10, s5
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v7, v1
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v9
; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v9, v1, s4
; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v1, v0
-; CGP-NEXT: v_mul_hi_u32 v12, v10, s5
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v11, 0
-; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v7
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v12, v[1:2]
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v9, v0
-; CGP-NEXT: v_subb_u32_e64 v1, s[4:5], v10, v7, vcc
-; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v10, v7
+; CGP-NEXT: v_mul_lo_u32 v8, v1, s5
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v8, v7
+; CGP-NEXT: v_mul_hi_u32 v12, v1, s5
+; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v11, 0
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v12, v9
+; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v12, v[8:9]
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
+; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v1, v9, vcc
+; CGP-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v9
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v6
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CGP-NEXT: v_cndmask_b32_e64 v1, -1, v8, s[4:5]
-; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT: v_cndmask_b32_e64 v7, -1, v8, s[4:5]
+; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v11
; CGP-NEXT: s_add_u32 s4, s8, s9
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s10
-; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v6
; CGP-NEXT: s_cselect_b32 s4, 1, 0
-; CGP-NEXT: v_add_i32_e32 v8, vcc, 1, v11
+; CGP-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; CGP-NEXT: s_add_i32 s5, s5, s4
-; CGP-NEXT: v_addc_u32_e32 v9, vcc, 0, v12, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v8
; CGP-NEXT: s_add_u32 s4, s11, s13
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v6
+; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v9, vcc
; CGP-NEXT: s_cselect_b32 s8, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s14
-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; CGP-NEXT: s_cselect_b32 s9, 1, 0
-; CGP-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, 1, v8
+; CGP-NEXT: v_cndmask_b32_e32 v0, v8, v1, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
+; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
; CGP-NEXT: s_add_i32 s8, s8, s9
-; CGP-NEXT: v_addc_u32_e32 v10, vcc, 0, v9, vcc
-; CGP-NEXT: s_add_u32 s4, s4, s5
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CGP-NEXT: s_cselect_b32 s5, 1, 0
-; CGP-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; CGP-NEXT: s_add_i32 s8, s8, s5
; CGP-NEXT: v_cndmask_b32_e32 v0, v11, v0, vcc
-; CGP-NEXT: s_add_i32 s12, s12, s8
-; CGP-NEXT: v_cndmask_b32_e32 v1, v12, v7, vcc
+; CGP-NEXT: s_add_u32 s4, s4, s5
+; CGP-NEXT: v_cndmask_b32_e32 v1, v12, v1, vcc
; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
-; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT: v_mul_hi_u32 v4, v4, s4
+; CGP-NEXT: s_add_i32 s8, s8, s5
; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
+; CGP-NEXT: s_add_i32 s12, s12, s8
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: s_add_u32 s4, s6, s4
; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_mov_b32_e32 v2, s6
+; CGP-NEXT: v_mul_hi_u32 v2, v4, s4
+; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: v_mov_b32_e32 v4, s4
; CGP-NEXT: s_addc_u32 s5, s7, s12
-; CGP-NEXT: v_readfirstlane_b32 s7, v4
-; CGP-NEXT: v_mul_hi_u32 v4, s4, v2
+; CGP-NEXT: v_readfirstlane_b32 s7, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v4, s6
; CGP-NEXT: s_mul_i32 s8, s5, 0xffed2705
; CGP-NEXT: s_add_u32 s7, s8, s7
; CGP-NEXT: s_mul_i32 s8, s4, -1
; CGP-NEXT: s_add_u32 s7, s8, s7
-; CGP-NEXT: v_readfirstlane_b32 s8, v4
-; CGP-NEXT: v_mov_b32_e32 v4, s7
+; CGP-NEXT: v_mov_b32_e32 v5, s6
+; CGP-NEXT: v_readfirstlane_b32 s8, v2
; CGP-NEXT: s_mul_i32 s9, s4, s7
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
-; CGP-NEXT: v_mul_hi_u32 v5, s4, v4
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v5
+; CGP-NEXT: v_mul_hi_u32 v4, v4, s7
; CGP-NEXT: s_mul_i32 s6, s5, s6
; CGP-NEXT: s_add_u32 s8, s8, s9
; CGP-NEXT: s_cselect_b32 s9, 1, 0
@@ -1684,13 +1688,14 @@ define <2 x i64> @v_sdiv_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: s_cselect_b32 s6, 1, 0
; CGP-NEXT: s_add_i32 s9, s9, s6
; CGP-NEXT: v_readfirstlane_b32 s6, v2
-; CGP-NEXT: v_readfirstlane_b32 s8, v5
+; CGP-NEXT: v_readfirstlane_b32 s8, v4
+; CGP-NEXT: v_mov_b32_e32 v2, s7
; CGP-NEXT: s_mul_i32 s7, s5, s7
; CGP-NEXT: s_add_u32 s6, s6, s8
; CGP-NEXT: s_cselect_b32 s8, 1, 0
; CGP-NEXT: s_add_u32 s6, s6, s7
; CGP-NEXT: s_cselect_b32 s7, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v4
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
; CGP-NEXT: s_add_i32 s7, s8, s7
; CGP-NEXT: s_add_u32 s6, s6, s9
; CGP-NEXT: s_cselect_b32 s8, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index b7120b5f2e39b..6af79882b1747 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -243,29 +243,29 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: s_add_i32 s17, s17, s18
; GFX8-NEXT: s_add_u32 s15, s16, s15
; GFX8-NEXT: v_mov_b32_e32 v0, s15
-; GFX8-NEXT: v_mul_hi_u32 v0, s12, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s12, v0
; GFX8-NEXT: s_mul_i32 s16, s12, s15
; GFX8-NEXT: s_addc_u32 s14, s14, s17
; GFX8-NEXT: s_mul_i32 s12, s12, s14
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v1, s15, v0
+; GFX8-NEXT: v_readfirstlane_b32 s17, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s16
; GFX8-NEXT: s_add_u32 s12, s12, s17
; GFX8-NEXT: s_mul_i32 s13, s13, s15
; GFX8-NEXT: s_add_u32 s12, s13, s12
+; GFX8-NEXT: v_mov_b32_e32 v2, s16
; GFX8-NEXT: v_readfirstlane_b32 s13, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
; GFX8-NEXT: s_mul_i32 s17, s15, s12
-; GFX8-NEXT: v_mul_hi_u32 v0, s14, v0
-; GFX8-NEXT: v_mul_hi_u32 v2, s15, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s14, v2
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s12
; GFX8-NEXT: s_mul_i32 s16, s14, s16
; GFX8-NEXT: s_add_u32 s13, s13, s17
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_u32 s13, s13, s16
; GFX8-NEXT: s_cselect_b32 s13, 1, 0
; GFX8-NEXT: s_add_i32 s17, s17, s13
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: v_readfirstlane_b32 s16, v2
+; GFX8-NEXT: v_readfirstlane_b32 s13, v1
+; GFX8-NEXT: v_readfirstlane_b32 s16, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s12
; GFX8-NEXT: s_mul_i32 s12, s14, s12
; GFX8-NEXT: s_add_u32 s13, s13, s16
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
@@ -282,12 +282,12 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: v_mov_b32_e32 v0, s12
; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
; GFX8-NEXT: s_addc_u32 s13, s14, s16
-; GFX8-NEXT: v_mov_b32_e32 v1, s13
+; GFX8-NEXT: v_mov_b32_e32 v2, s13
; GFX8-NEXT: s_mul_i32 s15, s6, s13
; GFX8-NEXT: v_readfirstlane_b32 s14, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s7
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s12
-; GFX8-NEXT: v_mul_hi_u32 v2, s6, v1
+; GFX8-NEXT: v_mul_hi_u32 v3, s6, v2
; GFX8-NEXT: s_mul_i32 s16, s7, s12
; GFX8-NEXT: s_add_u32 s14, s14, s15
; GFX8-NEXT: s_cselect_b32 s15, 1, 0
@@ -295,17 +295,17 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: s_cselect_b32 s12, 1, 0
; GFX8-NEXT: s_add_i32 s15, s15, s12
; GFX8-NEXT: v_readfirstlane_b32 s12, v0
-; GFX8-NEXT: v_readfirstlane_b32 s14, v2
+; GFX8-NEXT: v_readfirstlane_b32 s14, v3
; GFX8-NEXT: s_mul_i32 s13, s7, s13
; GFX8-NEXT: s_add_u32 s12, s12, s14
; GFX8-NEXT: s_cselect_b32 s14, 1, 0
; GFX8-NEXT: s_add_u32 s12, s12, s13
; GFX8-NEXT: s_cselect_b32 s13, 1, 0
; GFX8-NEXT: s_add_i32 s13, s14, s13
+; GFX8-NEXT: v_mul_hi_u32 v0, s7, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s4
; GFX8-NEXT: s_add_u32 s12, s12, s15
-; GFX8-NEXT: v_mul_hi_u32 v0, s7, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s12
-; GFX8-NEXT: v_mul_hi_u32 v1, s4, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v1, s12
; GFX8-NEXT: s_cselect_b32 s14, 1, 0
; GFX8-NEXT: s_add_i32 s13, s13, s14
; GFX8-NEXT: v_readfirstlane_b32 s14, v0
@@ -1637,29 +1637,29 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_i32 s21, s21, s22
; GFX8-NEXT: s_add_u32 s19, s20, s19
; GFX8-NEXT: v_mov_b32_e32 v0, s19
-; GFX8-NEXT: v_mul_hi_u32 v0, s16, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s16, v0
; GFX8-NEXT: s_mul_i32 s20, s16, s19
; GFX8-NEXT: s_addc_u32 s18, s18, s21
; GFX8-NEXT: s_mul_i32 s16, s16, s18
-; GFX8-NEXT: v_readfirstlane_b32 s21, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: v_mul_hi_u32 v1, s19, v0
+; GFX8-NEXT: v_readfirstlane_b32 s21, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s20
; GFX8-NEXT: s_add_u32 s16, s16, s21
; GFX8-NEXT: s_mul_i32 s17, s17, s19
; GFX8-NEXT: s_add_u32 s16, s17, s16
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: v_readfirstlane_b32 s17, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
; GFX8-NEXT: s_mul_i32 s21, s19, s16
-; GFX8-NEXT: v_mul_hi_u32 v0, s18, v0
-; GFX8-NEXT: v_mul_hi_u32 v2, s19, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s18, v2
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
; GFX8-NEXT: s_mul_i32 s20, s18, s20
; GFX8-NEXT: s_add_u32 s17, s17, s21
; GFX8-NEXT: s_cselect_b32 s21, 1, 0
; GFX8-NEXT: s_add_u32 s17, s17, s20
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_i32 s21, s21, s17
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: v_readfirstlane_b32 s20, v2
+; GFX8-NEXT: v_readfirstlane_b32 s17, v1
+; GFX8-NEXT: v_readfirstlane_b32 s20, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s16
; GFX8-NEXT: s_mul_i32 s16, s18, s16
; GFX8-NEXT: s_add_u32 s17, s17, s20
; GFX8-NEXT: s_cselect_b32 s20, 1, 0
@@ -1676,12 +1676,12 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mov_b32_e32 v0, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
; GFX8-NEXT: s_addc_u32 s17, s18, s20
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
+; GFX8-NEXT: v_mov_b32_e32 v2, s17
; GFX8-NEXT: s_mul_i32 s19, s8, s17
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s9
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v2, s8, v1
+; GFX8-NEXT: v_mul_hi_u32 v3, s8, v2
; GFX8-NEXT: s_mul_i32 s20, s9, s16
; GFX8-NEXT: s_add_u32 s18, s18, s19
; GFX8-NEXT: s_cselect_b32 s19, 1, 0
@@ -1689,17 +1689,17 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
; GFX8-NEXT: s_add_i32 s19, s19, s16
; GFX8-NEXT: v_readfirstlane_b32 s16, v0
-; GFX8-NEXT: v_readfirstlane_b32 s18, v2
+; GFX8-NEXT: v_readfirstlane_b32 s18, v3
; GFX8-NEXT: s_mul_i32 s17, s9, s17
; GFX8-NEXT: s_add_u32 s16, s16, s18
; GFX8-NEXT: s_cselect_b32 s18, 1, 0
; GFX8-NEXT: s_add_u32 s16, s16, s17
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_i32 s17, s18, s17
+; GFX8-NEXT: v_mul_hi_u32 v0, s9, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s4
; GFX8-NEXT: s_add_u32 s16, s16, s19
-; GFX8-NEXT: v_mul_hi_u32 v0, s9, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: v_mul_hi_u32 v1, s4, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v1, s16
; GFX8-NEXT: s_cselect_b32 s18, 1, 0
; GFX8-NEXT: s_add_i32 s17, s17, s18
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
@@ -1817,29 +1817,29 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_i32 s21, s21, s22
; GFX8-NEXT: s_add_u32 s16, s16, s20
; GFX8-NEXT: v_mov_b32_e32 v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v0, s17, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s17, v0
; GFX8-NEXT: s_mul_i32 s20, s17, s16
; GFX8-NEXT: s_addc_u32 s18, s18, s21
; GFX8-NEXT: s_mul_i32 s17, s17, s18
-; GFX8-NEXT: v_readfirstlane_b32 s21, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: v_mul_hi_u32 v1, s16, v0
+; GFX8-NEXT: v_readfirstlane_b32 s21, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s20
; GFX8-NEXT: s_add_u32 s17, s17, s21
; GFX8-NEXT: s_mul_i32 s19, s19, s16
; GFX8-NEXT: s_add_u32 s17, s19, s17
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: v_readfirstlane_b32 s19, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
; GFX8-NEXT: s_mul_i32 s21, s16, s17
-; GFX8-NEXT: v_mul_hi_u32 v0, s18, v0
-; GFX8-NEXT: v_mul_hi_u32 v2, s16, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s18, v2
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s17
; GFX8-NEXT: s_mul_i32 s20, s18, s20
; GFX8-NEXT: s_add_u32 s19, s19, s21
; GFX8-NEXT: s_cselect_b32 s21, 1, 0
; GFX8-NEXT: s_add_u32 s19, s19, s20
; GFX8-NEXT: s_cselect_b32 s19, 1, 0
; GFX8-NEXT: s_add_i32 s21, s21, s19
-; GFX8-NEXT: v_readfirstlane_b32 s19, v0
-; GFX8-NEXT: v_readfirstlane_b32 s20, v2
+; GFX8-NEXT: v_readfirstlane_b32 s19, v1
+; GFX8-NEXT: v_readfirstlane_b32 s20, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s17
; GFX8-NEXT: s_mul_i32 s17, s18, s17
; GFX8-NEXT: s_add_u32 s19, s19, s20
; GFX8-NEXT: s_cselect_b32 s20, 1, 0
@@ -1856,12 +1856,12 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mov_b32_e32 v0, s16
; GFX8-NEXT: v_mul_hi_u32 v0, s10, v0
; GFX8-NEXT: s_addc_u32 s17, s18, s20
-; GFX8-NEXT: v_mov_b32_e32 v1, s17
+; GFX8-NEXT: v_mov_b32_e32 v2, s17
; GFX8-NEXT: s_mul_i32 s19, s10, s17
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s11
; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v2, s10, v1
+; GFX8-NEXT: v_mul_hi_u32 v3, s10, v2
; GFX8-NEXT: s_mul_i32 s20, s11, s16
; GFX8-NEXT: s_add_u32 s18, s18, s19
; GFX8-NEXT: s_cselect_b32 s19, 1, 0
@@ -1869,17 +1869,17 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
; GFX8-NEXT: s_add_i32 s19, s19, s16
; GFX8-NEXT: v_readfirstlane_b32 s16, v0
-; GFX8-NEXT: v_readfirstlane_b32 s18, v2
+; GFX8-NEXT: v_readfirstlane_b32 s18, v3
; GFX8-NEXT: s_mul_i32 s17, s11, s17
; GFX8-NEXT: s_add_u32 s16, s16, s18
; GFX8-NEXT: s_cselect_b32 s18, 1, 0
; GFX8-NEXT: s_add_u32 s16, s16, s17
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_i32 s17, s18, s17
+; GFX8-NEXT: v_mul_hi_u32 v0, s11, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: s_add_u32 s16, s16, s19
-; GFX8-NEXT: v_mul_hi_u32 v0, s11, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s16
-; GFX8-NEXT: v_mul_hi_u32 v1, s6, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v1, s16
; GFX8-NEXT: s_cselect_b32 s18, 1, 0
; GFX8-NEXT: s_add_i32 s17, s17, s18
; GFX8-NEXT: v_readfirstlane_b32 s18, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
index 55e38ee0134b2..feb93cb1d83e1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
@@ -258,10 +258,9 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v4, v2
-; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v5, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v0, v3, vcc
; GFX8-NEXT: flat_store_dword v[2:3], v1
; GFX8-NEXT: s_endpgm
;
@@ -273,10 +272,9 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v0, v3, vcc
; GFX9-NEXT: global_store_dword v[2:3], v1, off
; GFX9-NEXT: s_endpgm
;
@@ -288,10 +286,8 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX10-NEXT: v_mul_u32_u24_e32 v0, 7, v0
; GFX10-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, s0
-; GFX10-NEXT: v_mov_b32_e32 v5, s1
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v5, v3, vcc_lo
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, s0, v2
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, s1, v3, vcc_lo
; GFX10-NEXT: global_store_dword v[2:3], v1, off
; GFX10-NEXT: s_endpgm
;
@@ -302,9 +298,8 @@ define amdgpu_kernel void @mulu24_shl64(ptr addrspace(1) nocapture %arg) {
; GFX11-NEXT: v_mul_u32_u24_e32 v0, 7, v0
; GFX11-NEXT: v_lshlrev_b64 v[2:3], 2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
-; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2
-; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, v5, v3, vcc_lo
+; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, s0, v2
+; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v3, vcc_lo
; GFX11-NEXT: global_store_b32 v[2:3], v1, off
; GFX11-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index a739ede299c06..6440940eceae4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -253,29 +253,29 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_i32 s13, s13, s14
; CHECK-NEXT: s_add_u32 s11, s11, s12
; CHECK-NEXT: v_mov_b32_e32 v0, s11
-; CHECK-NEXT: v_mul_hi_u32 v0, s3, v0
+; CHECK-NEXT: v_mul_hi_u32 v1, s3, v0
; CHECK-NEXT: s_mul_i32 s12, s3, s11
; CHECK-NEXT: s_addc_u32 s10, s10, s13
; CHECK-NEXT: s_mul_i32 s3, s3, s10
-; CHECK-NEXT: v_readfirstlane_b32 s13, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s12
-; CHECK-NEXT: v_mul_hi_u32 v1, s11, v0
+; CHECK-NEXT: v_readfirstlane_b32 s13, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, v0, s12
; CHECK-NEXT: s_add_u32 s3, s3, s13
; CHECK-NEXT: s_mul_i32 s5, s5, s11
; CHECK-NEXT: s_add_u32 s3, s5, s3
+; CHECK-NEXT: v_mov_b32_e32 v2, s12
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: s_mul_i32 s13, s11, s3
-; CHECK-NEXT: v_mul_hi_u32 v0, s10, v0
-; CHECK-NEXT: v_mul_hi_u32 v2, s11, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, s10, v2
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
; CHECK-NEXT: s_mul_i32 s12, s10, s12
; CHECK-NEXT: s_add_u32 s5, s5, s13
; CHECK-NEXT: s_cselect_b32 s13, 1, 0
; CHECK-NEXT: s_add_u32 s5, s5, s12
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s13, s13, s5
-; CHECK-NEXT: v_readfirstlane_b32 s5, v0
-; CHECK-NEXT: v_readfirstlane_b32 s12, v2
+; CHECK-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-NEXT: v_readfirstlane_b32 s12, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: s_mul_i32 s3, s10, s3
; CHECK-NEXT: s_add_u32 s5, s5, s12
; CHECK-NEXT: s_cselect_b32 s12, 1, 0
@@ -292,12 +292,12 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_mov_b32_e32 v0, s3
; CHECK-NEXT: v_mul_hi_u32 v0, s8, v0
; CHECK-NEXT: s_addc_u32 s5, s10, s12
-; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mov_b32_e32 v2, s5
; CHECK-NEXT: s_mul_i32 s11, s8, s5
; CHECK-NEXT: v_readfirstlane_b32 s10, v0
; CHECK-NEXT: v_mov_b32_e32 v0, s9
; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
-; CHECK-NEXT: v_mul_hi_u32 v2, s8, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, s8, v2
; CHECK-NEXT: s_mul_i32 s12, s9, s3
; CHECK-NEXT: s_add_u32 s10, s10, s11
; CHECK-NEXT: s_cselect_b32 s11, 1, 0
@@ -305,17 +305,17 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_cselect_b32 s3, 1, 0
; CHECK-NEXT: s_add_i32 s11, s11, s3
; CHECK-NEXT: v_readfirstlane_b32 s3, v0
-; CHECK-NEXT: v_readfirstlane_b32 s10, v2
+; CHECK-NEXT: v_readfirstlane_b32 s10, v3
; CHECK-NEXT: s_mul_i32 s5, s9, s5
; CHECK-NEXT: s_add_u32 s3, s3, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
; CHECK-NEXT: s_add_u32 s3, s3, s5
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s5, s10, s5
+; CHECK-NEXT: v_mul_hi_u32 v0, s9, v2
+; CHECK-NEXT: v_mov_b32_e32 v1, s6
; CHECK-NEXT: s_add_u32 s3, s3, s11
-; CHECK-NEXT: v_mul_hi_u32 v0, s9, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s3
-; CHECK-NEXT: v_mul_hi_u32 v1, s6, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, v1, s3
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
; CHECK-NEXT: s_add_i32 s5, s5, s10
; CHECK-NEXT: v_readfirstlane_b32 s10, v0
@@ -1076,24 +1076,24 @@ define i64 @v_srem_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_cvt_f32_u32_e32 v3, 0x12d8fb
; CHECK-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, 0xffed2705
; CHECK-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
; CHECK-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT: v_xor_b32_e32 v5, v0, v2
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; CHECK-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; CHECK-NEXT: v_trunc_f32_e32 v4, v4
; CHECK-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
; CHECK-NEXT: v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v4
-; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
-; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
-; CHECK-NEXT: v_mul_hi_u32 v6, v4, v3
+; CHECK-NEXT: v_cvt_u32_f32_e32 v4, v4
+; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
+; CHECK-NEXT: v_mul_hi_u32 v6, v5, v3
; CHECK-NEXT: v_readfirstlane_b32 s4, v3
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
-; CHECK-NEXT: v_readfirstlane_b32 s5, v0
+; CHECK-NEXT: v_readfirstlane_b32 s5, v4
; CHECK-NEXT: v_readfirstlane_b32 s7, v6
; CHECK-NEXT: v_mul_hi_u32 v6, v3, s6
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
@@ -1102,7 +1102,7 @@ define i64 @v_srem_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: v_readfirstlane_b32 s8, v6
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v6, v0, s6
+; CHECK-NEXT: v_mul_hi_u32 v6, v4, s6
; CHECK-NEXT: v_mul_hi_u32 v3, v3, s7
; CHECK-NEXT: s_mul_i32 s10, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
@@ -1117,97 +1117,98 @@ define i64 @v_srem_i64_oddk_denom(i64 %num) {
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_u32 s6, s6, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v0, v0, s7
+; CHECK-NEXT: v_mul_hi_u32 v3, v4, s7
; CHECK-NEXT: s_add_i32 s8, s8, s10
; CHECK-NEXT: s_add_u32 s6, s6, s9
; CHECK-NEXT: s_cselect_b32 s7, 1, 0
; CHECK-NEXT: s_add_i32 s8, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s7, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v3
; CHECK-NEXT: s_add_i32 s7, s7, s8
; CHECK-NEXT: s_add_u32 s4, s4, s6
-; CHECK-NEXT: v_mul_hi_u32 v0, v4, s4
+; CHECK-NEXT: v_mul_hi_u32 v3, v5, s4
; CHECK-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CHECK-NEXT: v_mov_b32_e32 v4, s4
; CHECK-NEXT: s_addc_u32 s5, s5, s7
-; CHECK-NEXT: v_xor_b32_e32 v6, v1, v2
-; CHECK-NEXT: v_readfirstlane_b32 s7, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s6
-; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v3
+; CHECK-NEXT: v_mul_hi_u32 v3, v4, s6
; CHECK-NEXT: s_mul_i32 s8, s5, 0xffed2705
; CHECK-NEXT: s_add_u32 s7, s8, s7
; CHECK-NEXT: s_mul_i32 s8, s4, -1
; CHECK-NEXT: s_add_u32 s7, s8, s7
-; CHECK-NEXT: v_readfirstlane_b32 s8, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s7
+; CHECK-NEXT: v_mov_b32_e32 v5, s6
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v0, s5, v0
-; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, s5, v5
+; CHECK-NEXT: v_mul_hi_u32 v4, v4, s7
; CHECK-NEXT: s_mul_i32 s6, s5, s6
; CHECK-NEXT: s_add_u32 s8, s8, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_u32 s6, s8, s6
; CHECK-NEXT: s_cselect_b32 s6, 1, 0
; CHECK-NEXT: s_add_i32 s9, s9, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v0
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: v_readfirstlane_b32 s6, v3
+; CHECK-NEXT: v_readfirstlane_b32 s8, v4
+; CHECK-NEXT: v_mov_b32_e32 v3, s7
; CHECK-NEXT: s_mul_i32 s7, s5, s7
; CHECK-NEXT: s_add_u32 s6, s6, s8
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_u32 s6, s6, s7
; CHECK-NEXT: s_cselect_b32 s7, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v0, s5, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, s5, v3
; CHECK-NEXT: s_add_i32 s7, s8, s7
; CHECK-NEXT: s_add_u32 s6, s6, s9
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_i32 s7, s7, s8
-; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: v_readfirstlane_b32 s8, v3
; CHECK-NEXT: s_add_i32 s8, s8, s7
; CHECK-NEXT: s_add_u32 s4, s4, s6
; CHECK-NEXT: s_addc_u32 s5, s5, s8
-; CHECK-NEXT: v_mul_hi_u32 v0, v5, s4
-; CHECK-NEXT: v_mul_lo_u32 v1, v5, s5
-; CHECK-NEXT: v_mul_lo_u32 v3, v6, s4
-; CHECK-NEXT: v_mul_hi_u32 v4, v5, s5
-; CHECK-NEXT: v_mul_hi_u32 v9, v6, s5
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CHECK-NEXT: v_mul_hi_u32 v3, v6, s4
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; CHECK-NEXT: v_mul_lo_u32 v1, v6, s5
+; CHECK-NEXT: v_mul_hi_u32 v3, v0, s4
+; CHECK-NEXT: v_mul_lo_u32 v4, v0, s5
+; CHECK-NEXT: v_mul_lo_u32 v5, v1, s4
+; CHECK-NEXT: v_mul_hi_u32 v6, v0, s5
+; CHECK-NEXT: v_mov_b32_e32 v7, 0x12d8fb
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v5
; CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; CHECK-NEXT: v_mul_hi_u32 v5, v1, s4
; CHECK-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v1, v0
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v4, 0
+; CHECK-NEXT: v_mul_lo_u32 v4, v1, s5
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, v4, v3
+; CHECK-NEXT: v_mul_hi_u32 v9, v1, s5
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v6, 0
; CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v3
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v8, v[1:2]
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v5, v0
-; CHECK-NEXT: v_subb_u32_e64 v1, s[4:5], v6, v3, vcc
-; CHECK-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
-; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, v9, v5
+; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v8, v[4:5]
+; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
+; CHECK-NEXT: v_subb_u32_e64 v3, s[4:5], v1, v5, vcc
+; CHECK-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v5
+; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CHECK-NEXT: v_sub_i32_e32 v5, vcc, v0, v7
-; CHECK-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v5, v7
; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; CHECK-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v7
; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
; CHECK-NEXT: v_sub_i32_e32 v7, vcc, v5, v7
; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CHECK-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v3, vcc
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
; CHECK-NEXT: v_cndmask_b32_e64 v4, -1, v4, s[4:5]
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
; CHECK-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
; CHECK-NEXT: v_xor_b32_e32 v1, v1, v2
; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
@@ -1221,188 +1222,189 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-LABEL: v_srem_v2i64_oddk_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_cvt_f32_u32_e32 v4, 0x12d8fb
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0
-; GISEL-NEXT: s_mov_b32 s8, 1
-; GISEL-NEXT: s_cmp_lg_u32 s8, 0
-; GISEL-NEXT: v_mac_f32_e32 v4, 0x4f800000, v5
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v5, 0x12d8fb
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v6, 0
; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
-; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
-; GISEL-NEXT: v_trunc_f32_e32 v6, v6
-; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
-; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v5
-; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v6
-; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v1, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v0, v4
+; GISEL-NEXT: v_mac_f32_e32 v5, 0x4f800000, v6
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v5
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, v1, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v8, v0, v4
+; GISEL-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v5
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x2f800000, v1
+; GISEL-NEXT: v_trunc_f32_e32 v5, v5
+; GISEL-NEXT: v_mac_f32_e32 v1, 0xcf800000, v5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v6, v1
; GISEL-NEXT: v_mov_b32_e32 v0, 0xffed2705
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, v7
-; GISEL-NEXT: v_readfirstlane_b32 s7, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v1, v5
+; GISEL-NEXT: s_mov_b32 s8, 1
+; GISEL-NEXT: v_mul_hi_u32 v5, v0, v6
+; GISEL-NEXT: v_readfirstlane_b32 s7, v6
; GISEL-NEXT: s_mul_i32 s5, s7, 0xffed2705
-; GISEL-NEXT: v_readfirstlane_b32 s6, v5
-; GISEL-NEXT: v_readfirstlane_b32 s9, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, v7, s5
+; GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GISEL-NEXT: v_readfirstlane_b32 s9, v5
+; GISEL-NEXT: v_mul_hi_u32 v5, v6, s5
; GISEL-NEXT: s_subb_u32 s4, 0, 0
; GISEL-NEXT: s_mul_i32 s10, s6, 0xffed2705
; GISEL-NEXT: s_add_u32 s9, s10, s9
; GISEL-NEXT: s_mul_i32 s10, s4, s7
; GISEL-NEXT: s_add_u32 s10, s10, s9
-; GISEL-NEXT: v_readfirstlane_b32 s11, v8
+; GISEL-NEXT: v_readfirstlane_b32 s11, v5
; GISEL-NEXT: s_mul_i32 s12, s7, s10
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s5
-; GISEL-NEXT: v_mul_hi_u32 v9, v7, s10
+; GISEL-NEXT: v_mul_hi_u32 v5, v1, s5
+; GISEL-NEXT: v_mul_hi_u32 v9, v6, s10
; GISEL-NEXT: s_mul_i32 s13, s6, s5
; GISEL-NEXT: s_add_u32 s12, s11, s12
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-NEXT: s_add_u32 s12, s12, s13
; GISEL-NEXT: s_cselect_b32 s5, 1, 0
; GISEL-NEXT: s_add_i32 s14, s14, s5
-; GISEL-NEXT: v_readfirstlane_b32 s12, v8
+; GISEL-NEXT: v_readfirstlane_b32 s12, v5
; GISEL-NEXT: v_readfirstlane_b32 s5, v9
; GISEL-NEXT: s_mul_i32 s15, s6, s10
; GISEL-NEXT: s_add_u32 s5, s12, s5
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-NEXT: s_add_u32 s5, s5, s15
; GISEL-NEXT: s_cselect_b32 s15, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v8, v5, s10
+; GISEL-NEXT: v_mul_hi_u32 v5, v1, s10
; GISEL-NEXT: s_add_i32 s15, s16, s15
; GISEL-NEXT: s_add_u32 s5, s5, s14
; GISEL-NEXT: s_cselect_b32 s10, 1, 0
; GISEL-NEXT: s_add_i32 s15, s15, s10
-; GISEL-NEXT: v_readfirstlane_b32 s10, v8
+; GISEL-NEXT: v_readfirstlane_b32 s10, v5
; GISEL-NEXT: s_add_i32 s10, s10, s15
; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_mul_hi_u32 v8, v0, s5
+; GISEL-NEXT: v_mul_hi_u32 v5, v0, s5
; GISEL-NEXT: s_mul_i32 s14, s5, 0xffed2705
-; GISEL-NEXT: v_xor_b32_e32 v14, v6, v4
-; GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GISEL-NEXT: v_mov_b32_e32 v9, s5
; GISEL-NEXT: s_addc_u32 s10, s6, s10
-; GISEL-NEXT: v_readfirstlane_b32 s15, v8
-; GISEL-NEXT: v_mul_hi_u32 v8, s5, v6
+; GISEL-NEXT: v_readfirstlane_b32 s15, v5
+; GISEL-NEXT: v_mul_hi_u32 v5, v9, s14
; GISEL-NEXT: s_mul_i32 s16, s10, 0xffed2705
; GISEL-NEXT: s_add_u32 s15, s16, s15
; GISEL-NEXT: s_mul_i32 s4, s4, s5
; GISEL-NEXT: s_add_u32 s4, s4, s15
-; GISEL-NEXT: v_readfirstlane_b32 s15, v8
-; GISEL-NEXT: v_mov_b32_e32 v8, s4
+; GISEL-NEXT: v_mov_b32_e32 v10, s14
+; GISEL-NEXT: v_readfirstlane_b32 s15, v5
; GISEL-NEXT: s_mul_i32 s16, s5, s4
-; GISEL-NEXT: v_mul_hi_u32 v6, s10, v6
-; GISEL-NEXT: v_mul_hi_u32 v9, s5, v8
+; GISEL-NEXT: v_mul_hi_u32 v5, s10, v10
+; GISEL-NEXT: v_mul_hi_u32 v9, v9, s4
; GISEL-NEXT: s_mul_i32 s14, s10, s14
; GISEL-NEXT: s_add_u32 s15, s15, s16
; GISEL-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-NEXT: s_add_u32 s14, s15, s14
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-NEXT: s_add_i32 s16, s16, s14
-; GISEL-NEXT: v_readfirstlane_b32 s14, v6
+; GISEL-NEXT: v_readfirstlane_b32 s14, v5
; GISEL-NEXT: v_readfirstlane_b32 s15, v9
+; GISEL-NEXT: v_mov_b32_e32 v5, s4
; GISEL-NEXT: s_mul_i32 s4, s10, s4
; GISEL-NEXT: s_add_u32 s14, s14, s15
; GISEL-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-NEXT: s_add_u32 s4, s14, s4
; GISEL-NEXT: s_cselect_b32 s14, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v6, s10, v8
+; GISEL-NEXT: v_mul_hi_u32 v5, s10, v5
; GISEL-NEXT: s_add_i32 s14, s15, s14
; GISEL-NEXT: s_add_u32 s4, s4, s16
; GISEL-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-NEXT: s_add_i32 s14, s14, s15
-; GISEL-NEXT: v_readfirstlane_b32 s15, v6
+; GISEL-NEXT: v_readfirstlane_b32 s15, v5
; GISEL-NEXT: s_add_i32 s15, s15, s14
; GISEL-NEXT: s_add_u32 s4, s5, s4
; GISEL-NEXT: s_addc_u32 s5, s10, s15
-; GISEL-NEXT: v_mul_hi_u32 v8, v1, s4
-; GISEL-NEXT: v_mul_lo_u32 v9, v1, s5
-; GISEL-NEXT: v_mul_lo_u32 v10, v14, s4
-; GISEL-NEXT: v_mul_hi_u32 v11, v1, s5
-; GISEL-NEXT: v_mov_b32_e32 v6, 0x12d8fb
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_xor_b32_e32 v7, v7, v4
+; GISEL-NEXT: v_mul_hi_u32 v9, v8, s4
+; GISEL-NEXT: v_mul_lo_u32 v10, v8, s5
+; GISEL-NEXT: v_mul_lo_u32 v11, v7, s4
+; GISEL-NEXT: v_mul_hi_u32 v12, v8, s5
+; GISEL-NEXT: v_mov_b32_e32 v5, 0x12d8fb
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v10, v14, s4
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_mul_lo_u32 v9, v14, s5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v11, v7, s4
; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v10, v7, s5
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v9, v8
-; GISEL-NEXT: v_mul_hi_u32 v12, v14, s5
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v6, v15, 0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v10
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v12, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v8
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v10, v9
+; GISEL-NEXT: v_mul_hi_u32 v13, v7, s5
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v5, v15, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v11
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v5, v13, v[10:11]
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: s_cmp_lg_u32 s8, 0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], 0, v15, v[10:11]
-; GISEL-NEXT: v_subb_u32_e64 v8, s[4:5], v14, v12, vcc
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v14, v12
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v1, v6
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], 0, v15, v[11:12]
+; GISEL-NEXT: v_subb_u32_e64 v9, s[4:5], v7, v13, vcc
+; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], v7, v13
+; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v8, v5
+; GISEL-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v12, -1, v12, vcc
+; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v11, v5
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v9
+; GISEL-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v7, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v10, -1, v10, s[4:5]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v7, v14, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
; GISEL-NEXT: s_subb_u32 s4, 0, 0
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
+; GISEL-NEXT: v_xor_b32_e32 v11, v7, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v3
; GISEL-NEXT: s_mul_i32 s5, s4, s7
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v7
; GISEL-NEXT: s_add_u32 s5, s5, s9
+; GISEL-NEXT: v_xor_b32_e32 v12, v2, v7
; GISEL-NEXT: s_mul_i32 s8, s7, s5
-; GISEL-NEXT: v_mul_hi_u32 v7, v7, s5
+; GISEL-NEXT: v_mul_hi_u32 v2, v6, s5
; GISEL-NEXT: s_add_u32 s8, s11, s8
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-NEXT: s_add_u32 s8, s8, s13
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-NEXT: s_add_i32 s9, s9, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-NEXT: s_mul_i32 s10, s6, s5
; GISEL-NEXT: s_add_u32 s8, s12, s8
; GISEL-NEXT: s_cselect_b32 s11, 1, 0
; GISEL-NEXT: s_add_u32 s8, s8, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
; GISEL-NEXT: s_cselect_b32 s10, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v5, v5, s5
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v1, v6
+; GISEL-NEXT: v_mul_hi_u32 v1, v1, s5
; GISEL-NEXT: s_add_i32 s10, s11, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
; GISEL-NEXT: s_add_u32 s5, s8, s9
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v6
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9
; GISEL-NEXT: s_add_i32 s10, s10, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v12, -1, v12, vcc
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v11, v6
+; GISEL-NEXT: v_readfirstlane_b32 s8, v1
; GISEL-NEXT: s_add_i32 s8, s8, s10
-; GISEL-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v9, vcc
; GISEL-NEXT: s_add_u32 s5, s7, s5
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GISEL-NEXT: v_mul_hi_u32 v0, v0, s5
-; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v14, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; GISEL-NEXT: v_xor_b32_e32 v11, v8, v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
; GISEL-NEXT: s_mul_i32 s7, s5, 0xffed2705
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
; GISEL-NEXT: s_addc_u32 s6, s6, s8
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GISEL-NEXT: v_mov_b32_e32 v0, s7
-; GISEL-NEXT: v_xor_b32_e32 v5, v2, v8
-; GISEL-NEXT: v_mul_hi_u32 v2, s5, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v1, s7
; GISEL-NEXT: s_mul_i32 s9, s6, 0xffed2705
; GISEL-NEXT: s_add_u32 s8, s9, s8
; GISEL-NEXT: s_mul_i32 s4, s4, s5
; GISEL-NEXT: s_add_u32 s4, s4, s8
-; GISEL-NEXT: v_readfirstlane_b32 s8, v2
-; GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: s_mul_i32 s9, s5, s4
-; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
-; GISEL-NEXT: v_mul_hi_u32 v7, s5, v2
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
+; GISEL-NEXT: v_mul_hi_u32 v1, v1, s4
; GISEL-NEXT: s_mul_i32 s7, s6, s7
; GISEL-NEXT: s_add_u32 s8, s8, s9
; GISEL-NEXT: s_cselect_b32 s9, 1, 0
@@ -1410,13 +1412,14 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: s_cselect_b32 s7, 1, 0
; GISEL-NEXT: s_add_i32 s9, s9, s7
; GISEL-NEXT: v_readfirstlane_b32 s7, v0
-; GISEL-NEXT: v_readfirstlane_b32 s8, v7
+; GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
; GISEL-NEXT: s_mul_i32 s4, s6, s4
; GISEL-NEXT: s_add_u32 s7, s7, s8
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-NEXT: s_add_u32 s4, s7, s4
; GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GISEL-NEXT: v_mul_hi_u32 v0, s6, v2
+; GISEL-NEXT: v_mul_hi_u32 v0, s6, v0
; GISEL-NEXT: s_add_i32 s7, s8, s7
; GISEL-NEXT: s_add_u32 s4, s4, s9
; GISEL-NEXT: s_cselect_b32 s8, 1, 0
@@ -1424,62 +1427,62 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GISEL-NEXT: s_add_i32 s8, s8, s7
; GISEL-NEXT: s_add_u32 s4, s5, s4
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v7, vcc
; GISEL-NEXT: s_addc_u32 s5, s6, s8
-; GISEL-NEXT: v_mul_hi_u32 v0, v5, s4
-; GISEL-NEXT: v_mul_lo_u32 v2, v5, s5
-; GISEL-NEXT: v_xor_b32_e32 v7, v3, v8
-; GISEL-NEXT: v_mul_lo_u32 v3, v7, s4
-; GISEL-NEXT: v_mul_hi_u32 v9, v5, s5
+; GISEL-NEXT: v_mul_hi_u32 v0, v12, s4
+; GISEL-NEXT: v_mul_lo_u32 v1, v12, s5
+; GISEL-NEXT: v_xor_b32_e32 v6, v3, v7
+; GISEL-NEXT: v_mul_lo_u32 v2, v6, s4
+; GISEL-NEXT: v_mul_hi_u32 v3, v12, s5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v3, v7, s4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; GISEL-NEXT: v_mul_lo_u32 v2, v7, s5
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_mul_hi_u32 v2, v6, s4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v6, s5
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v3
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v2, v0
-; GISEL-NEXT: v_mul_hi_u32 v10, v7, s5
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v12, 0
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v2, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_xor_b32_e32 v10, v8, v4
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v3, v2
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v1, v0
+; GISEL-NEXT: v_mul_hi_u32 v1, v6, s5
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v13, 0
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v9, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v0, v[3:4]
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v1, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v0, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v10, v4
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v11, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v12, v[9:10]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
-; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v7, v3, vcc
-; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v7, v3
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], 0, v13, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v12, v2
+; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], v6, v3, vcc
+; GISEL-NEXT: v_sub_i32_e64 v3, s[4:5], v6, v3
; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT: v_sub_i32_e32 v7, vcc, v2, v6
+; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v2, v5
; GISEL-NEXT: v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v7, v6
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v8, v5
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v5
; GISEL-NEXT: v_cndmask_b32_e32 v9, -1, v9, vcc
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v5, vcc, v8, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
; GISEL-NEXT: v_subbrev_u32_e32 v10, vcc, 0, v3, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v5, -1, v5, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v6, -1, v6, s[4:5]
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v8, v5, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v8
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v8
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v8
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v7
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v7
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v7
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v7, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_srem_v2i64_oddk_denom:
@@ -1491,29 +1494,29 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v5
; CGP-NEXT: v_mac_f32_e32 v4, 0x4f800000, v6
; CGP-NEXT: v_rcp_iflag_f32_e32 v4, v4
-; CGP-NEXT: v_xor_b32_e32 v9, v0, v5
; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
; CGP-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
; CGP-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
; CGP-NEXT: v_trunc_f32_e32 v6, v6
; CGP-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
; CGP-NEXT: v_cvt_u32_f32_e32 v7, v4
; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705
-; CGP-NEXT: v_cvt_u32_f32_e32 v0, v6
-; CGP-NEXT: v_xor_b32_e32 v10, v1, v5
-; CGP-NEXT: v_mul_hi_u32 v6, v4, v7
+; CGP-NEXT: v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
+; CGP-NEXT: v_mul_hi_u32 v8, v4, v7
; CGP-NEXT: v_readfirstlane_b32 s6, v7
; CGP-NEXT: s_mul_i32 s4, s6, 0xffed2705
-; CGP-NEXT: v_readfirstlane_b32 s7, v0
-; CGP-NEXT: v_readfirstlane_b32 s5, v6
-; CGP-NEXT: v_mul_hi_u32 v6, v7, s4
+; CGP-NEXT: v_readfirstlane_b32 s7, v6
+; CGP-NEXT: v_readfirstlane_b32 s5, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v7, s4
; CGP-NEXT: s_mul_i32 s8, s7, 0xffed2705
; CGP-NEXT: s_add_u32 s5, s8, s5
; CGP-NEXT: s_mul_i32 s8, s6, -1
; CGP-NEXT: s_add_u32 s5, s8, s5
-; CGP-NEXT: v_readfirstlane_b32 s8, v6
+; CGP-NEXT: v_readfirstlane_b32 s8, v8
; CGP-NEXT: s_mul_i32 s9, s6, s5
-; CGP-NEXT: v_mul_hi_u32 v6, v0, s4
+; CGP-NEXT: v_mul_hi_u32 v8, v6, s4
; CGP-NEXT: v_mul_hi_u32 v7, v7, s5
; CGP-NEXT: s_mul_i32 s10, s7, s4
; CGP-NEXT: s_add_u32 s11, s8, s9
@@ -1521,141 +1524,142 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: s_add_u32 s11, s11, s10
; CGP-NEXT: s_cselect_b32 s4, 1, 0
; CGP-NEXT: s_add_i32 s12, s12, s4
-; CGP-NEXT: v_readfirstlane_b32 s11, v6
+; CGP-NEXT: v_readfirstlane_b32 s11, v8
; CGP-NEXT: v_readfirstlane_b32 s13, v7
; CGP-NEXT: s_mul_i32 s14, s7, s5
; CGP-NEXT: s_add_u32 s4, s11, s13
; CGP-NEXT: s_cselect_b32 s15, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s14
; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v0, v0, s5
+; CGP-NEXT: v_mul_hi_u32 v6, v6, s5
; CGP-NEXT: s_add_i32 s15, s15, s16
; CGP-NEXT: s_add_u32 s4, s4, s12
; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: s_add_i32 s15, s15, s5
-; CGP-NEXT: v_readfirstlane_b32 s12, v0
+; CGP-NEXT: v_readfirstlane_b32 s12, v6
; CGP-NEXT: s_add_i32 s5, s12, s15
; CGP-NEXT: s_add_u32 s4, s6, s4
-; CGP-NEXT: v_mul_hi_u32 v0, v4, s4
+; CGP-NEXT: v_mul_hi_u32 v6, v4, s4
; CGP-NEXT: s_mul_i32 s15, s4, 0xffed2705
+; CGP-NEXT: v_mov_b32_e32 v7, s4
; CGP-NEXT: s_addc_u32 s5, s7, s5
+; CGP-NEXT: v_readfirstlane_b32 s16, v6
+; CGP-NEXT: v_mul_hi_u32 v6, v7, s15
; CGP-NEXT: s_mul_i32 s17, s5, 0xffed2705
-; CGP-NEXT: v_readfirstlane_b32 s16, v0
-; CGP-NEXT: v_mov_b32_e32 v0, s15
-; CGP-NEXT: v_mul_hi_u32 v1, s4, v0
; CGP-NEXT: s_add_u32 s16, s17, s16
; CGP-NEXT: s_mul_i32 s17, s4, -1
; CGP-NEXT: s_add_u32 s16, s17, s16
-; CGP-NEXT: v_readfirstlane_b32 s17, v1
-; CGP-NEXT: v_mov_b32_e32 v1, s16
+; CGP-NEXT: v_mov_b32_e32 v8, s15
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
; CGP-NEXT: s_mul_i32 s18, s4, s16
-; CGP-NEXT: v_mul_hi_u32 v0, s5, v0
-; CGP-NEXT: v_mul_hi_u32 v6, s4, v1
+; CGP-NEXT: v_mul_hi_u32 v6, s5, v8
+; CGP-NEXT: v_mul_hi_u32 v7, v7, s16
; CGP-NEXT: s_mul_i32 s15, s5, s15
; CGP-NEXT: s_add_u32 s17, s17, s18
; CGP-NEXT: s_cselect_b32 s18, 1, 0
; CGP-NEXT: s_add_u32 s15, s17, s15
; CGP-NEXT: s_cselect_b32 s15, 1, 0
; CGP-NEXT: s_add_i32 s18, s18, s15
-; CGP-NEXT: v_readfirstlane_b32 s15, v0
-; CGP-NEXT: v_readfirstlane_b32 s17, v6
+; CGP-NEXT: v_readfirstlane_b32 s15, v6
+; CGP-NEXT: v_readfirstlane_b32 s17, v7
+; CGP-NEXT: v_mov_b32_e32 v6, s16
; CGP-NEXT: s_mul_i32 s16, s5, s16
; CGP-NEXT: s_add_u32 s15, s15, s17
; CGP-NEXT: s_cselect_b32 s17, 1, 0
; CGP-NEXT: s_add_u32 s15, s15, s16
; CGP-NEXT: s_cselect_b32 s16, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v0, s5, v1
+; CGP-NEXT: v_mul_hi_u32 v6, s5, v6
; CGP-NEXT: s_add_i32 s16, s17, s16
; CGP-NEXT: s_add_u32 s15, s15, s18
; CGP-NEXT: s_cselect_b32 s17, 1, 0
; CGP-NEXT: s_add_i32 s16, s16, s17
-; CGP-NEXT: v_readfirstlane_b32 s17, v0
+; CGP-NEXT: v_readfirstlane_b32 s17, v6
; CGP-NEXT: s_add_i32 s17, s17, s16
; CGP-NEXT: s_add_u32 s4, s4, s15
; CGP-NEXT: s_addc_u32 s5, s5, s17
-; CGP-NEXT: v_mul_hi_u32 v0, v9, s4
-; CGP-NEXT: v_mul_lo_u32 v1, v9, s5
-; CGP-NEXT: v_mul_lo_u32 v7, v10, s4
-; CGP-NEXT: v_mul_hi_u32 v8, v9, s5
+; CGP-NEXT: v_mul_hi_u32 v7, v0, s4
+; CGP-NEXT: v_mul_lo_u32 v8, v0, s5
+; CGP-NEXT: v_mul_lo_u32 v9, v1, s4
+; CGP-NEXT: v_mul_hi_u32 v10, v0, s5
; CGP-NEXT: v_mov_b32_e32 v6, 0x12d8fb
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CGP-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v7
-; CGP-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CGP-NEXT: v_mul_hi_u32 v7, v10, s4
-; CGP-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; CGP-NEXT: v_mul_lo_u32 v1, v10, s5
; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v1, vcc, v7, v1
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v9
; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_mul_hi_u32 v9, v1, s4
; CGP-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT: v_add_i32_e32 v8, vcc, v1, v0
-; CGP-NEXT: v_mul_hi_u32 v12, v10, s5
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v8, 0
+; CGP-NEXT: v_mul_lo_u32 v8, v1, s5
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v8, v7
+; CGP-NEXT: v_mul_hi_u32 v12, v1, s5
+; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v10, 0
; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v11
-; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v7
-; CGP-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v6, v11, v[1:2]
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v9, v0
-; CGP-NEXT: v_subb_u32_e64 v1, s[4:5], v10, v7, vcc
-; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v10, v7
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v12, v9
+; CGP-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v6, v11, v[8:9]
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
+; CGP-NEXT: v_subb_u32_e64 v7, s[4:5], v1, v9, vcc
+; CGP-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v9
; CGP-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v6
; CGP-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; CGP-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CGP-NEXT: v_cndmask_b32_e64 v8, -1, v8, s[4:5]
+; CGP-NEXT: v_sub_i32_e32 v9, vcc, v0, v6
; CGP-NEXT: s_add_u32 s4, s8, s9
+; CGP-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s10
-; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v9, v6
; CGP-NEXT: s_cselect_b32 s4, 1, 0
-; CGP-NEXT: v_sub_i32_e32 v9, vcc, v0, v6
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; CGP-NEXT: s_add_i32 s5, s5, s4
-; CGP-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v10, -1, v10, vcc
+; CGP-NEXT: v_sub_i32_e32 v11, vcc, v9, v6
; CGP-NEXT: s_add_u32 s4, s11, s13
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v9, v6
+; CGP-NEXT: v_subbrev_u32_e32 v12, vcc, 0, v1, vcc
; CGP-NEXT: s_cselect_b32 s8, 1, 0
; CGP-NEXT: s_add_u32 s4, s4, s14
-; CGP-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; CGP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; CGP-NEXT: s_cselect_b32 s9, 1, 0
-; CGP-NEXT: v_cndmask_b32_e32 v10, -1, v10, vcc
-; CGP-NEXT: v_sub_i32_e32 v11, vcc, v9, v6
-; CGP-NEXT: s_add_i32 s8, s8, s9
-; CGP-NEXT: v_subbrev_u32_e32 v12, vcc, 0, v7, vcc
-; CGP-NEXT: s_add_u32 s4, s4, s5
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; CGP-NEXT: s_cselect_b32 s5, 1, 0
+; CGP-NEXT: s_cselect_b32 s9, 1, 0
; CGP-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; CGP-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc
+; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
; CGP-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; CGP-NEXT: s_add_i32 s8, s8, s5
+; CGP-NEXT: s_add_i32 s8, s8, s9
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; CGP-NEXT: s_add_i32 s12, s12, s8
-; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc
+; CGP-NEXT: s_add_u32 s4, s4, s5
+; CGP-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
; CGP-NEXT: v_xor_b32_e32 v0, v0, v5
-; CGP-NEXT: s_add_u32 s4, s6, s4
+; CGP-NEXT: s_cselect_b32 s5, 1, 0
; CGP-NEXT: v_xor_b32_e32 v1, v1, v5
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT: v_mul_hi_u32 v4, v4, s4
+; CGP-NEXT: s_add_i32 s8, s8, s5
; CGP-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
; CGP-NEXT: v_ashrrev_i32_e32 v7, 31, v3
+; CGP-NEXT: s_add_i32 s12, s12, s8
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v7
-; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: s_add_u32 s4, s6, s4
; CGP-NEXT: v_xor_b32_e32 v8, v2, v7
-; CGP-NEXT: v_mov_b32_e32 v2, s6
+; CGP-NEXT: v_mul_hi_u32 v2, v4, s4
+; CGP-NEXT: s_mul_i32 s6, s4, 0xffed2705
+; CGP-NEXT: v_mov_b32_e32 v4, s4
; CGP-NEXT: s_addc_u32 s5, s7, s12
-; CGP-NEXT: v_readfirstlane_b32 s7, v4
-; CGP-NEXT: v_mul_hi_u32 v4, s4, v2
+; CGP-NEXT: v_readfirstlane_b32 s7, v2
+; CGP-NEXT: v_mul_hi_u32 v2, v4, s6
; CGP-NEXT: s_mul_i32 s8, s5, 0xffed2705
; CGP-NEXT: s_add_u32 s7, s8, s7
; CGP-NEXT: s_mul_i32 s8, s4, -1
; CGP-NEXT: s_add_u32 s7, s8, s7
-; CGP-NEXT: v_readfirstlane_b32 s8, v4
-; CGP-NEXT: v_mov_b32_e32 v4, s7
+; CGP-NEXT: v_mov_b32_e32 v5, s6
+; CGP-NEXT: v_readfirstlane_b32 s8, v2
; CGP-NEXT: s_mul_i32 s9, s4, s7
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
-; CGP-NEXT: v_mul_hi_u32 v5, s4, v4
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v5
+; CGP-NEXT: v_mul_hi_u32 v4, v4, s7
; CGP-NEXT: s_mul_i32 s6, s5, s6
; CGP-NEXT: s_add_u32 s8, s8, s9
; CGP-NEXT: s_cselect_b32 s9, 1, 0
@@ -1663,13 +1667,14 @@ define <2 x i64> @v_srem_v2i64_oddk_denom(<2 x i64> %num) {
; CGP-NEXT: s_cselect_b32 s6, 1, 0
; CGP-NEXT: s_add_i32 s9, s9, s6
; CGP-NEXT: v_readfirstlane_b32 s6, v2
-; CGP-NEXT: v_readfirstlane_b32 s8, v5
+; CGP-NEXT: v_readfirstlane_b32 s8, v4
+; CGP-NEXT: v_mov_b32_e32 v2, s7
; CGP-NEXT: s_mul_i32 s7, s5, s7
; CGP-NEXT: s_add_u32 s6, s6, s8
; CGP-NEXT: s_cselect_b32 s8, 1, 0
; CGP-NEXT: s_add_u32 s6, s6, s7
; CGP-NEXT: s_cselect_b32 s7, 1, 0
-; CGP-NEXT: v_mul_hi_u32 v2, s5, v4
+; CGP-NEXT: v_mul_hi_u32 v2, s5, v2
; CGP-NEXT: s_add_i32 s7, s8, s7
; CGP-NEXT: s_add_u32 s6, s6, s9
; CGP-NEXT: s_cselect_b32 s8, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index cfe86e2b7081a..b03fdfdd97b70 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -188,75 +188,73 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_or_b64 s[4:5], s[0:1], s[2:3]
; CHECK-NEXT: s_mov_b32 s4, 0
-; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s2
; CHECK-NEXT: v_cmp_ne_u64_e64 s[4:5], s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, 1
; CHECK-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
; CHECK-NEXT: s_cbranch_scc0 .LBB1_2
; CHECK-NEXT: ; %bb.1:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s2
; CHECK-NEXT: v_cvt_f32_u32_e32 v1, s3
; CHECK-NEXT: s_sub_u32 s4, 0, s2
; CHECK-NEXT: s_subb_u32 s5, 0, s3
-; CHECK-NEXT: v_madmk_f32 v1, v1, 0x4f800000, v0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1
-; CHECK-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
-; CHECK-NEXT: v_trunc_f32_e32 v2, v2
-; CHECK-NEXT: v_mac_f32_e32 v1, 0xcf800000, v2
+; CHECK-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; CHECK-NEXT: v_trunc_f32_e32 v1, v1
+; CHECK-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
-; CHECK-NEXT: v_readfirstlane_b32 s7, v1
+; CHECK-NEXT: v_mul_hi_u32 v2, s4, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_readfirstlane_b32 s6, v2
-; CHECK-NEXT: v_mul_hi_u32 v4, v1, s9
+; CHECK-NEXT: v_readfirstlane_b32 s6, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, v0, s9
; CHECK-NEXT: s_mul_i32 s8, s4, s6
-; CHECK-NEXT: v_readfirstlane_b32 s12, v3
+; CHECK-NEXT: v_readfirstlane_b32 s12, v2
; CHECK-NEXT: s_mul_i32 s10, s5, s7
; CHECK-NEXT: s_add_i32 s8, s12, s8
; CHECK-NEXT: s_add_i32 s8, s8, s10
-; CHECK-NEXT: v_readfirstlane_b32 s10, v4
+; CHECK-NEXT: v_readfirstlane_b32 s10, v3
; CHECK-NEXT: s_mul_i32 s12, s7, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, v2, s9
-; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
+; CHECK-NEXT: v_mul_hi_u32 v2, v1, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s8
; CHECK-NEXT: s_mul_i32 s11, s6, s9
; CHECK-NEXT: s_add_u32 s10, s10, s12
; CHECK-NEXT: s_cselect_b32 s12, 1, 0
; CHECK-NEXT: s_add_u32 s10, s10, s11
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_i32 s12, s12, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v3
-; CHECK-NEXT: v_readfirstlane_b32 s10, v1
+; CHECK-NEXT: v_readfirstlane_b32 s9, v2
+; CHECK-NEXT: v_readfirstlane_b32 s10, v0
; CHECK-NEXT: s_mul_i32 s11, s6, s8
; CHECK-NEXT: s_add_u32 s9, s9, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
; CHECK-NEXT: s_add_u32 s9, s9, s11
; CHECK-NEXT: s_cselect_b32 s11, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v1, v2, s8
+; CHECK-NEXT: v_mul_hi_u32 v0, v1, s8
; CHECK-NEXT: s_add_i32 s10, s10, s11
; CHECK-NEXT: s_add_u32 s8, s9, s12
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_i32 s10, s10, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v1
+; CHECK-NEXT: v_readfirstlane_b32 s9, v0
; CHECK-NEXT: s_add_i32 s9, s9, s10
; CHECK-NEXT: s_add_u32 s7, s7, s8
-; CHECK-NEXT: v_mov_b32_e32 v1, s7
-; CHECK-NEXT: v_mul_hi_u32 v1, s4, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, s7
+; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
; CHECK-NEXT: s_mul_i32 s8, s4, s7
; CHECK-NEXT: s_addc_u32 s6, s6, s9
; CHECK-NEXT: s_mul_i32 s4, s4, s6
; CHECK-NEXT: v_readfirstlane_b32 s9, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s8
-; CHECK-NEXT: v_mul_hi_u32 v1, s7, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, v0, s8
; CHECK-NEXT: s_mul_i32 s5, s5, s7
; CHECK-NEXT: s_add_i32 s4, s9, s4
; CHECK-NEXT: s_add_i32 s4, s4, s5
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s6
-; CHECK-NEXT: v_mov_b32_e32 v2, s4
; CHECK-NEXT: s_mul_i32 s9, s7, s4
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, s7, v2
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s4
; CHECK-NEXT: s_mul_i32 s10, s6, s8
; CHECK-NEXT: s_add_u32 s5, s5, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
@@ -264,28 +262,29 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s9, s9, s5
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, s4
; CHECK-NEXT: s_mul_i32 s4, s6, s4
; CHECK-NEXT: s_add_u32 s5, s5, s8
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_u32 s4, s5, s4
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v1, s6, v2
+; CHECK-NEXT: v_mul_hi_u32 v0, s6, v1
; CHECK-NEXT: s_add_i32 s5, s8, s5
; CHECK-NEXT: s_add_u32 s4, s4, s9
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_i32 s5, s5, s8
-; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
; CHECK-NEXT: s_add_i32 s8, s8, s5
; CHECK-NEXT: s_add_u32 s4, s7, s4
-; CHECK-NEXT: v_mov_b32_e32 v1, s4
-; CHECK-NEXT: v_mul_hi_u32 v1, s0, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
+; CHECK-NEXT: v_mul_hi_u32 v0, s0, v0
; CHECK-NEXT: s_addc_u32 s5, s6, s8
; CHECK-NEXT: v_mov_b32_e32 v2, s5
; CHECK-NEXT: s_mul_i32 s7, s0, s5
-; CHECK-NEXT: v_readfirstlane_b32 s6, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s1
-; CHECK-NEXT: v_mul_hi_u32 v1, v1, s4
+; CHECK-NEXT: v_readfirstlane_b32 s6, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s1
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s4
; CHECK-NEXT: v_mul_hi_u32 v3, s0, v2
; CHECK-NEXT: s_mul_i32 s8, s1, s4
; CHECK-NEXT: s_add_u32 s6, s6, s7
@@ -293,7 +292,7 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_u32 s6, s6, s8
; CHECK-NEXT: s_cselect_b32 s4, 1, 0
; CHECK-NEXT: s_add_i32 s7, s7, s4
-; CHECK-NEXT: v_readfirstlane_b32 s4, v1
+; CHECK-NEXT: v_readfirstlane_b32 s4, v0
; CHECK-NEXT: v_readfirstlane_b32 s6, v3
; CHECK-NEXT: s_mul_i32 s5, s1, s5
; CHECK-NEXT: s_add_u32 s4, s4, s6
@@ -301,15 +300,15 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_u32 s4, s4, s5
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s5, s6, s5
+; CHECK-NEXT: v_mul_hi_u32 v0, s1, v2
+; CHECK-NEXT: v_mov_b32_e32 v1, s2
; CHECK-NEXT: s_add_u32 s4, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v1, s1, v2
-; CHECK-NEXT: v_mov_b32_e32 v2, s4
-; CHECK-NEXT: v_mul_hi_u32 v2, s2, v2
+; CHECK-NEXT: v_mul_hi_u32 v1, v1, s4
; CHECK-NEXT: s_cselect_b32 s6, 1, 0
; CHECK-NEXT: s_add_i32 s5, s5, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v1
+; CHECK-NEXT: v_readfirstlane_b32 s6, v0
; CHECK-NEXT: s_add_i32 s5, s6, s5
-; CHECK-NEXT: v_readfirstlane_b32 s7, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v1
; CHECK-NEXT: s_mul_i32 s8, s2, s5
; CHECK-NEXT: s_mul_i32 s9, s3, s4
; CHECK-NEXT: s_add_i32 s7, s7, s8
@@ -354,8 +353,9 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_cmp_lg_u32 s1, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB1_5
; CHECK-NEXT: ; %bb.4:
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s2
; CHECK-NEXT: s_sub_i32 s1, 0, s2
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
; CHECK-NEXT: v_readfirstlane_b32 s3, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index 4202200db53d9..6085d9db3d6e0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -199,29 +199,29 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: s_add_i32 s13, s13, s14
; GFX8-NEXT: s_add_u32 s11, s12, s11
; GFX8-NEXT: v_mov_b32_e32 v0, s11
-; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s8, v0
; GFX8-NEXT: s_mul_i32 s12, s8, s11
; GFX8-NEXT: s_addc_u32 s10, s10, s13
; GFX8-NEXT: s_mul_i32 s8, s8, s10
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
-; GFX8-NEXT: v_mul_hi_u32 v1, s11, v0
+; GFX8-NEXT: v_readfirstlane_b32 s13, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s12
; GFX8-NEXT: s_add_u32 s8, s8, s13
; GFX8-NEXT: s_mul_i32 s9, s9, s11
; GFX8-NEXT: s_add_u32 s8, s9, s8
+; GFX8-NEXT: v_mov_b32_e32 v2, s12
; GFX8-NEXT: v_readfirstlane_b32 s9, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: s_mul_i32 s13, s11, s8
-; GFX8-NEXT: v_mul_hi_u32 v0, s10, v0
-; GFX8-NEXT: v_mul_hi_u32 v2, s11, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s10, v2
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s8
; GFX8-NEXT: s_mul_i32 s12, s10, s12
; GFX8-NEXT: s_add_u32 s9, s9, s13
; GFX8-NEXT: s_cselect_b32 s13, 1, 0
; GFX8-NEXT: s_add_u32 s9, s9, s12
; GFX8-NEXT: s_cselect_b32 s9, 1, 0
; GFX8-NEXT: s_add_i32 s13, s13, s9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: v_readfirstlane_b32 s12, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s12, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: s_mul_i32 s8, s10, s8
; GFX8-NEXT: s_add_u32 s9, s9, s12
; GFX8-NEXT: s_cselect_b32 s12, 1, 0
@@ -1340,30 +1340,30 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_i32 s17, s2, s3
; GFX8-NEXT: s_add_u32 s18, s1, s0
; GFX8-NEXT: v_mov_b32_e32 v0, s18
-; GFX8-NEXT: v_mul_hi_u32 v0, s10, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s10, v0
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX8-NEXT: s_mul_i32 s9, s10, s18
; GFX8-NEXT: s_addc_u32 s8, s16, s17
-; GFX8-NEXT: v_readfirstlane_b32 s16, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s9
-; GFX8-NEXT: v_mul_hi_u32 v1, s18, v0
+; GFX8-NEXT: v_readfirstlane_b32 s16, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s9
; GFX8-NEXT: s_mul_i32 s10, s10, s8
; GFX8-NEXT: s_add_u32 s10, s10, s16
; GFX8-NEXT: s_mul_i32 s11, s11, s18
; GFX8-NEXT: s_add_u32 s10, s11, s10
+; GFX8-NEXT: v_mov_b32_e32 v2, s9
; GFX8-NEXT: v_readfirstlane_b32 s11, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: s_mul_i32 s16, s18, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX8-NEXT: v_mul_hi_u32 v2, s18, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s8, v2
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s10
; GFX8-NEXT: s_mul_i32 s9, s8, s9
; GFX8-NEXT: s_add_u32 s11, s11, s16
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
; GFX8-NEXT: s_add_u32 s9, s11, s9
; GFX8-NEXT: s_cselect_b32 s9, 1, 0
; GFX8-NEXT: s_add_i32 s16, s16, s9
-; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: v_readfirstlane_b32 s11, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: s_mul_i32 s10, s8, s10
; GFX8-NEXT: s_add_u32 s9, s9, s11
; GFX8-NEXT: s_cselect_b32 s11, 1, 0
@@ -1505,29 +1505,29 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: s_add_i32 s17, s17, s18
; GFX8-NEXT: s_add_u32 s10, s10, s16
; GFX8-NEXT: v_mov_b32_e32 v0, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, s11, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s11, v0
; GFX8-NEXT: s_mul_i32 s16, s11, s10
; GFX8-NEXT: s_addc_u32 s12, s12, s17
; GFX8-NEXT: s_mul_i32 s11, s11, s12
-; GFX8-NEXT: v_readfirstlane_b32 s17, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s16
-; GFX8-NEXT: v_mul_hi_u32 v1, s10, v0
+; GFX8-NEXT: v_readfirstlane_b32 s17, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, v0, s16
; GFX8-NEXT: s_add_u32 s11, s11, s17
; GFX8-NEXT: s_mul_i32 s13, s13, s10
; GFX8-NEXT: s_add_u32 s11, s13, s11
+; GFX8-NEXT: v_mov_b32_e32 v2, s16
; GFX8-NEXT: v_readfirstlane_b32 s13, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s11
; GFX8-NEXT: s_mul_i32 s17, s10, s11
-; GFX8-NEXT: v_mul_hi_u32 v0, s12, v0
-; GFX8-NEXT: v_mul_hi_u32 v2, s10, v1
+; GFX8-NEXT: v_mul_hi_u32 v1, s12, v2
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s11
; GFX8-NEXT: s_mul_i32 s16, s12, s16
; GFX8-NEXT: s_add_u32 s13, s13, s17
; GFX8-NEXT: s_cselect_b32 s17, 1, 0
; GFX8-NEXT: s_add_u32 s13, s13, s16
; GFX8-NEXT: s_cselect_b32 s13, 1, 0
; GFX8-NEXT: s_add_i32 s17, s17, s13
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: v_readfirstlane_b32 s16, v2
+; GFX8-NEXT: v_readfirstlane_b32 s13, v1
+; GFX8-NEXT: v_readfirstlane_b32 s16, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s11
; GFX8-NEXT: s_mul_i32 s11, s12, s11
; GFX8-NEXT: s_add_u32 s13, s13, s16
; GFX8-NEXT: s_cselect_b32 s16, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 69dca2f91764c..421d8424c6f47 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -185,75 +185,73 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_or_b64 s[4:5], s[0:1], s[2:3]
; CHECK-NEXT: s_mov_b32 s4, 0
-; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s2
; CHECK-NEXT: v_cmp_ne_u64_e64 s[4:5], s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, 1
; CHECK-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
; CHECK-NEXT: s_cbranch_scc0 .LBB1_2
; CHECK-NEXT: ; %bb.1:
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s2
; CHECK-NEXT: v_cvt_f32_u32_e32 v1, s3
; CHECK-NEXT: s_sub_u32 s4, 0, s2
; CHECK-NEXT: s_subb_u32 s5, 0, s3
-; CHECK-NEXT: v_madmk_f32 v1, v1, 0x4f800000, v0
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; CHECK-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1
-; CHECK-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
-; CHECK-NEXT: v_trunc_f32_e32 v2, v2
-; CHECK-NEXT: v_mac_f32_e32 v1, 0xcf800000, v2
+; CHECK-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; CHECK-NEXT: v_trunc_f32_e32 v1, v1
+; CHECK-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2
-; CHECK-NEXT: v_mul_hi_u32 v3, s4, v1
-; CHECK-NEXT: v_readfirstlane_b32 s7, v1
+; CHECK-NEXT: v_mul_hi_u32 v2, s4, v0
+; CHECK-NEXT: v_readfirstlane_b32 s7, v0
; CHECK-NEXT: s_mul_i32 s9, s4, s7
-; CHECK-NEXT: v_readfirstlane_b32 s6, v2
-; CHECK-NEXT: v_mul_hi_u32 v4, v1, s9
+; CHECK-NEXT: v_readfirstlane_b32 s6, v1
+; CHECK-NEXT: v_mul_hi_u32 v3, v0, s9
; CHECK-NEXT: s_mul_i32 s8, s4, s6
-; CHECK-NEXT: v_readfirstlane_b32 s12, v3
+; CHECK-NEXT: v_readfirstlane_b32 s12, v2
; CHECK-NEXT: s_mul_i32 s10, s5, s7
; CHECK-NEXT: s_add_i32 s8, s12, s8
; CHECK-NEXT: s_add_i32 s8, s8, s10
-; CHECK-NEXT: v_readfirstlane_b32 s10, v4
+; CHECK-NEXT: v_readfirstlane_b32 s10, v3
; CHECK-NEXT: s_mul_i32 s12, s7, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, v2, s9
-; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
+; CHECK-NEXT: v_mul_hi_u32 v2, v1, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s8
; CHECK-NEXT: s_mul_i32 s11, s6, s9
; CHECK-NEXT: s_add_u32 s10, s10, s12
; CHECK-NEXT: s_cselect_b32 s12, 1, 0
; CHECK-NEXT: s_add_u32 s10, s10, s11
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_i32 s12, s12, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v3
-; CHECK-NEXT: v_readfirstlane_b32 s10, v1
+; CHECK-NEXT: v_readfirstlane_b32 s9, v2
+; CHECK-NEXT: v_readfirstlane_b32 s10, v0
; CHECK-NEXT: s_mul_i32 s11, s6, s8
; CHECK-NEXT: s_add_u32 s9, s9, s10
; CHECK-NEXT: s_cselect_b32 s10, 1, 0
; CHECK-NEXT: s_add_u32 s9, s9, s11
; CHECK-NEXT: s_cselect_b32 s11, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v1, v2, s8
+; CHECK-NEXT: v_mul_hi_u32 v0, v1, s8
; CHECK-NEXT: s_add_i32 s10, s10, s11
; CHECK-NEXT: s_add_u32 s8, s9, s12
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
; CHECK-NEXT: s_add_i32 s10, s10, s9
-; CHECK-NEXT: v_readfirstlane_b32 s9, v1
+; CHECK-NEXT: v_readfirstlane_b32 s9, v0
; CHECK-NEXT: s_add_i32 s9, s9, s10
; CHECK-NEXT: s_add_u32 s7, s7, s8
-; CHECK-NEXT: v_mov_b32_e32 v1, s7
-; CHECK-NEXT: v_mul_hi_u32 v1, s4, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, s7
+; CHECK-NEXT: v_mul_hi_u32 v1, s4, v0
; CHECK-NEXT: s_mul_i32 s8, s4, s7
; CHECK-NEXT: s_addc_u32 s6, s6, s9
; CHECK-NEXT: s_mul_i32 s4, s4, s6
; CHECK-NEXT: v_readfirstlane_b32 s9, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s8
-; CHECK-NEXT: v_mul_hi_u32 v1, s7, v1
+; CHECK-NEXT: v_mul_hi_u32 v1, v0, s8
; CHECK-NEXT: s_mul_i32 s5, s5, s7
; CHECK-NEXT: s_add_i32 s4, s9, s4
; CHECK-NEXT: s_add_i32 s4, s4, s5
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
; CHECK-NEXT: v_mov_b32_e32 v1, s6
-; CHECK-NEXT: v_mov_b32_e32 v2, s4
; CHECK-NEXT: s_mul_i32 s9, s7, s4
; CHECK-NEXT: v_mul_hi_u32 v1, v1, s8
-; CHECK-NEXT: v_mul_hi_u32 v3, s7, v2
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s4
; CHECK-NEXT: s_mul_i32 s10, s6, s8
; CHECK-NEXT: s_add_u32 s5, s5, s9
; CHECK-NEXT: s_cselect_b32 s9, 1, 0
@@ -261,28 +259,29 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s9, s9, s5
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
-; CHECK-NEXT: v_readfirstlane_b32 s8, v3
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, s4
; CHECK-NEXT: s_mul_i32 s4, s6, s4
; CHECK-NEXT: s_add_u32 s5, s5, s8
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_u32 s4, s5, s4
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
-; CHECK-NEXT: v_mul_hi_u32 v1, s6, v2
+; CHECK-NEXT: v_mul_hi_u32 v0, s6, v1
; CHECK-NEXT: s_add_i32 s5, s8, s5
; CHECK-NEXT: s_add_u32 s4, s4, s9
; CHECK-NEXT: s_cselect_b32 s8, 1, 0
; CHECK-NEXT: s_add_i32 s5, s5, s8
-; CHECK-NEXT: v_readfirstlane_b32 s8, v1
+; CHECK-NEXT: v_readfirstlane_b32 s8, v0
; CHECK-NEXT: s_add_i32 s8, s8, s5
; CHECK-NEXT: s_add_u32 s4, s7, s4
-; CHECK-NEXT: v_mov_b32_e32 v1, s4
-; CHECK-NEXT: v_mul_hi_u32 v1, s0, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
+; CHECK-NEXT: v_mul_hi_u32 v0, s0, v0
; CHECK-NEXT: s_addc_u32 s5, s6, s8
; CHECK-NEXT: v_mov_b32_e32 v2, s5
; CHECK-NEXT: s_mul_i32 s7, s0, s5
-; CHECK-NEXT: v_readfirstlane_b32 s6, v1
-; CHECK-NEXT: v_mov_b32_e32 v1, s1
-; CHECK-NEXT: v_mul_hi_u32 v1, v1, s4
+; CHECK-NEXT: v_readfirstlane_b32 s6, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s1
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s4
; CHECK-NEXT: v_mul_hi_u32 v3, s0, v2
; CHECK-NEXT: s_mul_i32 s8, s1, s4
; CHECK-NEXT: s_add_u32 s6, s6, s7
@@ -290,7 +289,7 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_u32 s6, s6, s8
; CHECK-NEXT: s_cselect_b32 s4, 1, 0
; CHECK-NEXT: s_add_i32 s7, s7, s4
-; CHECK-NEXT: v_readfirstlane_b32 s4, v1
+; CHECK-NEXT: v_readfirstlane_b32 s4, v0
; CHECK-NEXT: v_readfirstlane_b32 s6, v3
; CHECK-NEXT: s_mul_i32 s5, s1, s5
; CHECK-NEXT: s_add_u32 s4, s4, s6
@@ -298,15 +297,15 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_add_u32 s4, s4, s5
; CHECK-NEXT: s_cselect_b32 s5, 1, 0
; CHECK-NEXT: s_add_i32 s5, s6, s5
+; CHECK-NEXT: v_mul_hi_u32 v0, s1, v2
+; CHECK-NEXT: v_mov_b32_e32 v1, s2
; CHECK-NEXT: s_add_u32 s4, s4, s7
-; CHECK-NEXT: v_mul_hi_u32 v1, s1, v2
-; CHECK-NEXT: v_mov_b32_e32 v2, s4
-; CHECK-NEXT: v_mul_hi_u32 v2, s2, v2
+; CHECK-NEXT: v_mul_hi_u32 v1, v1, s4
; CHECK-NEXT: s_cselect_b32 s6, 1, 0
; CHECK-NEXT: s_add_i32 s5, s5, s6
-; CHECK-NEXT: v_readfirstlane_b32 s6, v1
+; CHECK-NEXT: v_readfirstlane_b32 s6, v0
; CHECK-NEXT: s_add_i32 s6, s6, s5
-; CHECK-NEXT: v_readfirstlane_b32 s7, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v1
; CHECK-NEXT: s_mul_i32 s6, s2, s6
; CHECK-NEXT: s_mul_i32 s5, s2, s4
; CHECK-NEXT: s_mul_i32 s4, s3, s4
@@ -353,8 +352,9 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: s_cmp_lg_u32 s1, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB1_5
; CHECK-NEXT: ; %bb.4:
-; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s2
; CHECK-NEXT: s_sub_i32 s1, 0, s2
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
; CHECK-NEXT: v_readfirstlane_b32 s3, v0
diff --git a/llvm/test/CodeGen/AMDGPU/bfi_int.ll b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
index 5bde70553ea9f..f2583da1e8e02 100644
--- a/llvm/test/CodeGen/AMDGPU/bfi_int.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
@@ -748,22 +748,18 @@ define amdgpu_ps <2 x float> @s_v_s_bitselect_i64_pat_0(i64 inreg %a, i64 %b, i6
;
; GFX8-GISEL-LABEL: s_v_s_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_and_b32_e32 v2, s0, v0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v3, s1, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: s_andn2_b64 s[0:1], s[2:3], s[0:1]
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_s_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_andn2_b64 s[2:3], s[2:3], s[0:1]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-GISEL-NEXT: v_and_or_b32 v0, s0, v0, v2
-; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, v3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, s0, v0, s2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, s3
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -800,26 +796,18 @@ define amdgpu_ps <2 x float> @s_s_v_bitselect_i64_pat_0(i64 inreg %a, i64 inreg
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_and_b64 s[2:3], s[0:1], s[2:3]
; GFX8-GISEL-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v2, v0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v3, v1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, s3, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_s_v_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_not_b64 s[4:5], s[0:1]
-; GFX10-GISEL-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s0
-; GFX10-GISEL-NEXT: v_and_or_b32 v1, v3, v1, v5
-; GFX10-GISEL-NEXT: v_and_or_b32 v0, v2, v0, v4
+; GFX10-GISEL-NEXT: s_and_b64 s[2:3], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_not_b64 s[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, s0, v0, s2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, s3
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -946,26 +934,22 @@ define amdgpu_ps <2 x float> @s_v_v_bitselect_i64_pat_0(i64 inreg %a, i64 %b, i6
;
; GFX8-GISEL-LABEL: s_v_v_bitselect_i64_pat_0:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: v_and_b32_e32 v4, s0, v0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v5, s1, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: s_not_b64 s[0:1], s[0:1]
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v2, s0, v2
+; GFX8-GISEL-NEXT: v_and_b32_e32 v3, s1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_v_bitselect_i64_pat_0:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_not_b64 s[2:3], s[0:1]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX10-GISEL-NEXT: v_and_b32_e32 v3, v5, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v2, v4, v2
-; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, s2, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, s3, v3
; GFX10-GISEL-NEXT: v_and_or_b32 v0, s0, v0, v2
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, s1, v1, v3
; GFX10-GISEL-NEXT: ; return to shader part epilog
%and0 = and i64 %a, %b
%not.a = xor i64 %a, -1
@@ -1128,10 +1112,8 @@ define amdgpu_ps <2 x float> @s_v_s_bitselect_i64_pat_1(i64 inreg %a, i64 %b, i6
; GFX8-GISEL-LABEL: s_v_s_bitselect_i64_pat_1:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v2, v0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v1, v3, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, s2, v0
; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, s3, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
@@ -1139,12 +1121,10 @@ define amdgpu_ps <2 x float> @s_v_s_bitselect_i64_pat_1(i64 inreg %a, i64 %b, i6
; GFX10-GISEL-LABEL: s_v_s_bitselect_i64_pat_1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, v3, v1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v2, v0
-; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, s3, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, s2, v0
+; GFX10-GISEL-NEXT: v_xor_b32_e32 v1, s3, v1
; GFX10-GISEL-NEXT: ; return to shader part epilog
%xor.0 = xor i64 %a, %mask
%and = and i64 %xor.0, %b
@@ -1331,26 +1311,18 @@ define amdgpu_ps <2 x float> @s_v_s_bfi_sha256_ma_i64(i64 inreg %x, i64 %y, i64
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_and_b64 s[4:5], s[0:1], s[2:3]
; GFX8-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-GISEL-NEXT: v_and_b32_e32 v2, v0, v2
-; GFX8-GISEL-NEXT: v_and_b32_e32 v3, v1, v3
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, s1, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, s5, v1
; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_v_s_bfi_sha256_ma_i64:
; GFX10-GISEL: ; %bb.0: ; %entry
-; GFX10-GISEL-NEXT: s_or_b64 s[4:5], s[0:1], s[2:3]
-; GFX10-GISEL-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s0
-; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, v3, v5
-; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, v2, v4
+; GFX10-GISEL-NEXT: s_and_b64 s[4:5], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, v0, s0, s4
+; GFX10-GISEL-NEXT: v_and_or_b32 v1, v1, s1, s5
; GFX10-GISEL-NEXT: ; return to shader part epilog
entry:
%and0 = and i64 %x, %z
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop.ll b/llvm/test/CodeGen/AMDGPU/ctpop.ll
index ec9bf36e76482..efd97635eb812 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop.ll
@@ -277,15 +277,14 @@ define amdgpu_kernel void @v_ctpop_add_chain_i32(ptr addrspace(1) noalias %out,
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v0, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1613,9 +1612,8 @@ define amdgpu_kernel void @v_ctpop_i32_add_vvar_inv(ptr addrspace(1) noalias %ou
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_dword v0, v[0:1]
; VI-GISEL-NEXT: s_mov_b32 s2, -1
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index e987b04b6e3bc..d1849cd6a1489 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -313,15 +313,14 @@ define amdgpu_kernel void @v_ctpop_add_chain_i16(ptr addrspace(1) noalias %out,
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 1, v0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_ushort v0, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -2139,9 +2138,8 @@ define amdgpu_kernel void @v_ctpop_i16_add_vvar_inv(ptr addrspace(1) noalias %ou
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_ushort v3, v[0:1]
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_ushort v0, v[0:1]
; VI-GISEL-NEXT: s_mov_b32 s2, -1
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index fe57a1ebda683..e04874032cacf 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -7424,11 +7424,11 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f64(ptr addrspace(1) %out)
; GFX6-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX6-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v4, s0
; GFX6-GISEL-NEXT: v_mov_b32_e32 v5, s1
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GFX6-GISEL-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v4, s0
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX6-GISEL-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
@@ -7463,11 +7463,11 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f64(ptr addrspace(1) %out)
; GFX8-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX8-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, s0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, v4, v0
+; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GFX8-GISEL-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, s0
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX8-GISEL-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index b0ebc333139a9..6898a02f04d98 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -36,8 +36,8 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -82,8 +82,8 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -132,10 +132,10 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xfffff800, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -181,8 +181,8 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -227,8 +227,8 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_rtn_2048:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -277,10 +277,10 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %v
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xfffff800, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -676,10 +676,9 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -761,13 +760,13 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -866,10 +865,9 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -957,12 +955,12 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -1055,12 +1053,10 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -1124,12 +1120,13 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -1211,12 +1208,10 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1286,14 +1281,14 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -1361,8 +1356,8 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_add_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -1410,10 +1405,10 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_add_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -1460,8 +1455,8 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_add_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -1508,10 +1503,10 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_add_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -1584,10 +1579,9 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -1670,13 +1664,13 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_add_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1778,10 +1772,9 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -1870,12 +1863,12 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_add_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -1974,12 +1967,10 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -2049,12 +2040,13 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_add_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -2144,12 +2136,10 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2225,14 +2215,14 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_add_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -2305,8 +2295,8 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -2354,10 +2344,10 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -2404,8 +2394,8 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -2452,10 +2442,10 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -2528,10 +2518,9 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -2616,13 +2605,13 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -2724,10 +2713,9 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -2818,12 +2806,12 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -2922,12 +2910,10 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -2999,12 +2985,13 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -3094,12 +3081,10 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3177,14 +3162,14 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_sub_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -3257,8 +3242,8 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_and_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -3306,10 +3291,10 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_and_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -3356,8 +3341,8 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_and_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -3404,10 +3389,10 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_and_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -3481,10 +3466,9 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -3569,13 +3553,13 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_and_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -3677,10 +3661,9 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -3771,12 +3754,12 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -3875,12 +3858,10 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -3952,12 +3933,13 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_and_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -4047,12 +4029,10 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4130,14 +4110,14 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -4209,8 +4189,8 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i3
;
; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -4258,10 +4238,10 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -4308,8 +4288,8 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -4356,10 +4336,10 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -4433,10 +4413,9 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -4521,13 +4500,13 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
;
; GFX950-GISEL-LABEL: flat_or_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -4629,10 +4608,9 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -4723,12 +4701,12 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
;
; GFX950-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -4827,12 +4805,10 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -4904,12 +4880,13 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_or_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -4999,12 +4976,10 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5082,14 +5057,14 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -5161,8 +5136,8 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -5210,10 +5185,10 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -5260,8 +5235,8 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc1
@@ -5308,10 +5283,10 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -5385,10 +5360,9 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -5473,13 +5447,13 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -5581,10 +5555,9 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -5675,12 +5648,12 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -5779,12 +5752,10 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -5856,12 +5827,13 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -5951,12 +5923,10 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6034,14 +6004,14 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -6106,8 +6076,8 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smax v0, v[2:3], v1 sc0
@@ -6145,10 +6115,10 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -6185,8 +6155,8 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v1
@@ -6223,10 +6193,10 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -6292,10 +6262,9 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -6375,13 +6344,13 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_max_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -6478,10 +6447,9 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -6567,12 +6535,12 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_max_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -6667,12 +6635,10 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -6739,12 +6705,13 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_max_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -6829,12 +6796,10 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6907,14 +6872,14 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_max_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -6978,8 +6943,8 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_min_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smin v0, v[2:3], v1 sc0
@@ -7017,10 +6982,10 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_min_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -7057,8 +7022,8 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_min_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smin v[2:3], v1
@@ -7095,10 +7060,10 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_min_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -7164,10 +7129,9 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -7247,13 +7211,13 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_min_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -7350,10 +7314,9 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -7439,12 +7402,12 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_min_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -7539,12 +7502,10 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -7611,12 +7572,13 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_min_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -7701,12 +7663,10 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7779,14 +7739,14 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_min_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -7850,8 +7810,8 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umax v0, v[2:3], v1 sc0
@@ -7889,10 +7849,10 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -7929,8 +7889,8 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umax v[2:3], v1
@@ -7967,10 +7927,10 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -8036,10 +7996,9 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -8119,13 +8078,13 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -8222,10 +8181,9 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -8311,12 +8269,12 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -8411,12 +8369,10 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -8483,12 +8439,13 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -8573,12 +8530,10 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8651,14 +8606,14 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -8722,8 +8677,8 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umin v0, v[2:3], v1 sc0
@@ -8761,10 +8716,10 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -8801,8 +8756,8 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umin v[2:3], v1
@@ -8839,10 +8794,10 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -8908,10 +8863,9 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -8991,13 +8945,13 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -9094,10 +9048,9 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -9183,12 +9136,12 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -9283,12 +9236,10 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -9355,12 +9306,13 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -9445,12 +9397,10 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9523,14 +9473,14 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -9602,11 +9552,11 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffse
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[0:1], v[2:3] sc0 sc1
@@ -9654,11 +9604,11 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
@@ -9707,11 +9657,11 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffs
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc1
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] sc1
@@ -9757,11 +9707,11 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
@@ -9836,11 +9786,10 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v0, v5
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v5
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v3 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -9929,14 +9878,13 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v4
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -10043,11 +9991,10 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v5
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v5
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, vcc_lo
@@ -10142,16 +10089,15 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v4
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
@@ -10252,14 +10198,12 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB92_3
@@ -10334,13 +10278,14 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v4
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -10434,17 +10379,15 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB93_3
@@ -10522,17 +10465,17 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
;
; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v3
; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v4
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -10598,8 +10541,8 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_inc v0, v[2:3], v1 sc0
@@ -10637,10 +10580,10 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -10675,8 +10618,8 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_inc v[2:3], v1
@@ -10710,10 +10653,10 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -10782,10 +10725,9 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -10872,13 +10814,13 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -10982,10 +10924,9 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -11078,12 +11019,12 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -11182,12 +11123,10 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -11257,12 +11196,13 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -11350,12 +11290,10 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -11431,14 +11369,14 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_inc_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
@@ -11505,8 +11443,8 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i32_rtn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_dec v0, v[2:3], v1 sc0
@@ -11544,10 +11482,10 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i32_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -11582,8 +11520,8 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i32_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_dec v[2:3], v1
@@ -11617,10 +11555,10 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i32_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_nop 1
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
@@ -11691,10 +11629,9 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
@@ -11784,13 +11721,13 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i64_rtn:
; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -11897,10 +11834,9 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
@@ -11996,12 +11932,12 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i64_rtn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
@@ -12103,12 +12039,10 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -12181,12 +12115,13 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i64_nortn:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
@@ -12277,12 +12212,10 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v1, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12361,14 +12294,14 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX950-GISEL-LABEL: flat_dec_saddr_i64_nortn_neg128:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s2, v0
+; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX950-GISEL-NEXT: s_nop 0
-; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v0
-; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index dd71d19d40493..5cb74e703e65e 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -575,12 +575,11 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %s
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, s0, s2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x800000, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -981,10 +980,10 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffse
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s0, v1, vcc_lo
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -1030,10 +1029,10 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s0, v1, vcc_lo
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs-v2f32.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs-v2f32.ll
index 25b6f48a0d964..a20f07eb0f009 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs-v2f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs-v2f32.ll
@@ -158,10 +158,8 @@ define <2 x float> @fneg_v2f32_s(<2 x float> inreg %first) {
; GFX1250-GISEL: ; %bb.0: ; %bb
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e64 v0, 0x80000000, s0
+; GFX1250-GISEL-NEXT: v_xor_b32_e64 v1, 0x80000000, s1
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
bb:
%neg = fneg <2 x float> %first
@@ -217,10 +215,8 @@ define <2 x float> @fabs_v2f32_s(<2 x float> inreg %first) {
; GFX1250-GISEL: ; %bb.0: ; %bb
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; GFX1250-GISEL-NEXT: v_and_b32_e64 v0, 0x7fffffff, s0
+; GFX1250-GISEL-NEXT: v_and_b32_e64 v1, 0x7fffffff, s1
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
bb:
%abs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %first)
@@ -286,10 +282,8 @@ define <2 x float> @fneg_fabs_v2f32_s(<2 x float> inreg %first) {
; GFX1250-GISEL: ; %bb.0: ; %bb
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
+; GFX1250-GISEL-NEXT: v_and_b32_e64 v0, 0x7fffffff, s0
+; GFX1250-GISEL-NEXT: v_and_b32_e64 v1, 0x7fffffff, s1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
diff --git a/llvm/test/CodeGen/AMDGPU/fold-readlane.mir b/llvm/test/CodeGen/AMDGPU/fold-readlane.mir
index 7caa7a4ef9db6..6e8a6a3324985 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-readlane.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-readlane.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu6.00 -run-pass si-fold-operands -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: name: fold-imm-readfirstlane{{$}}
@@ -371,8 +372,8 @@ body: |
# GCN-NEXT: %2:vgpr_32 = COPY %0
# GCN-NEXT: %3:vgpr_32 = COPY %1
# GCN-NEXT: %4:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, killed %3, %subreg.sub1
-# GCN-NEXT: %5:sreg_32_xm0 = V_READFIRSTLANE_B32 %4.sub0, implicit $exec
-# GCN-NEXT: %6:sreg_32_xm0 = V_READFIRSTLANE_B32 %4.sub1, implicit $exec
+# GCN-NEXT: %5:sreg_32_xm0 = COPY %0
+# GCN-NEXT: %6:sreg_32_xm0 = COPY %1
---
name: fold-copy-readfirstlane-regsequence1
tracksRegLiveness: true
@@ -386,4 +387,7 @@ body: |
%4:vreg_64 = REG_SEQUENCE %2:vgpr_32, %subreg.sub0, killed %3:vgpr_32, %subreg.sub1
%5:sreg_32_xm0 = V_READFIRSTLANE_B32 %4.sub0:vreg_64, implicit $exec
%6:sreg_32_xm0 = V_READFIRSTLANE_B32 %4.sub1:vreg_64, implicit $exec
+ S_NOP 0, implicit %5, implicit %6
...
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 5016fdf71f363..3a586e4746dd7 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -1981,8 +1981,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; GFX950-GISEL-NEXT: s_mov_b32 s2, -1
; GFX950-GISEL-NEXT: s_mov_b32 s3, 0xf000
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v0, v0, s5
; GFX950-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX950-GISEL-NEXT: s_endpgm
;
@@ -2116,11 +2116,9 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_pk_f16_f32 v0, s2, s3
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
;
@@ -2134,11 +2132,9 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_f16_f32 v0, s2, s3
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
index 1790832c9198c..2c88cf32e250f 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
@@ -714,14 +714,13 @@ define amdgpu_ps i32 @s_sqrt_f32(float inreg %x) {
; GISEL-IEEE-NEXT: s_cselect_b32 s3, s1, s0
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s3
; GISEL-IEEE-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s3
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-IEEE-NEXT: s_add_i32 s5, s4, -1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s5, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s6, s4, 1
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s3
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v2, s6
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v2, v0, s3
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s6, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_or_b64 s[0:1], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s4, s5, s4
@@ -831,14 +830,13 @@ define amdgpu_ps i32 @s_sqrt_f32_ninf(float inreg %x) {
; GISEL-IEEE-NEXT: s_cselect_b32 s3, s1, s0
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s3
; GISEL-IEEE-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s3
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-IEEE-NEXT: s_add_i32 s5, s4, -1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s5, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s6, s4, 1
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s3
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v2, s6
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v2, v0, s3
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s6, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_or_b64 s[0:1], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s4, s5, s4
@@ -4010,14 +4008,13 @@ define amdgpu_kernel void @elim_redun_check_neg0(ptr addrspace(1) %out, float %i
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s8
; GISEL-IEEE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GISEL-IEEE-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s8
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-IEEE-NEXT: s_add_i32 s5, s4, -1
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s5, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s9, s4, 1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s5
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s8
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v2, s9
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v2, v0, s8
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s9, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_or_b64 s[2:3], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s4, s5, s4
@@ -4164,14 +4161,13 @@ define amdgpu_kernel void @elim_redun_check_pos0(ptr addrspace(1) %out, float %i
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s8
; GISEL-IEEE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GISEL-IEEE-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s8
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-IEEE-NEXT: s_add_i32 s5, s4, -1
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s5, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s9, s4, 1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s5
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s8
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v2, s9
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v2, v0, s8
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s9, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_or_b64 s[2:3], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s4, s5, s4
@@ -4316,14 +4312,13 @@ define amdgpu_kernel void @elim_redun_check_ult(ptr addrspace(1) %out, float %in
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s8
; GISEL-IEEE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GISEL-IEEE-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s8
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-IEEE-NEXT: s_add_i32 s5, s4, -1
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s5, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s9, s4, 1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s5
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s8
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v2, s9
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v2, v0, s8
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s9, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_or_b64 s[2:3], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s4, s5, s4
@@ -4487,14 +4482,13 @@ define amdgpu_kernel void @elim_redun_check_v2(ptr addrspace(1) %out, <2 x float
; GISEL-IEEE-NEXT: s_cselect_b32 s7, s4, s2
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v2, s7
; GISEL-IEEE-NEXT: s_cselect_b32 s6, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v3, s7
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-IEEE-NEXT: s_add_i32 s9, s8, -1
; GISEL-IEEE-NEXT: s_add_i32 s10, s8, 1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v3, s9
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v4, s10
-; GISEL-IEEE-NEXT: v_fma_f32 v3, -v3, v2, s7
-; GISEL-IEEE-NEXT: v_fma_f32 v2, -v4, v2, s7
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v3
+; GISEL-IEEE-NEXT: v_fma_f32 v4, -s9, v2, v3
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s10, v2, v3
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v4
; GISEL-IEEE-NEXT: s_or_b64 s[4:5], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_cselect_b32 s8, s9, s8
@@ -4516,14 +4510,13 @@ define amdgpu_kernel void @elim_redun_check_v2(ptr addrspace(1) %out, <2 x float
; GISEL-IEEE-NEXT: s_cselect_b32 s8, s4, s3
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s8
; GISEL-IEEE-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s8
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s9, v0
; GISEL-IEEE-NEXT: s_add_i32 s10, s9, -1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s10
+; GISEL-IEEE-NEXT: v_fma_f32 v4, -s10, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s11, s9, 1
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s8
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v4, s11
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v4, v0, s8
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s11, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v4
; GISEL-IEEE-NEXT: s_or_b64 s[4:5], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s9, s10, s9
@@ -4734,14 +4727,13 @@ define amdgpu_kernel void @elim_redun_check_v2_ult(ptr addrspace(1) %out, <2 x f
; GISEL-IEEE-NEXT: s_cselect_b32 s7, s4, s2
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v2, s7
; GISEL-IEEE-NEXT: s_cselect_b32 s6, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v3, s7
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-IEEE-NEXT: s_add_i32 s9, s8, -1
; GISEL-IEEE-NEXT: s_add_i32 s10, s8, 1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v3, s9
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v4, s10
-; GISEL-IEEE-NEXT: v_fma_f32 v3, -v3, v2, s7
-; GISEL-IEEE-NEXT: v_fma_f32 v2, -v4, v2, s7
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v3
+; GISEL-IEEE-NEXT: v_fma_f32 v4, -s9, v2, v3
+; GISEL-IEEE-NEXT: v_fma_f32 v2, -s10, v2, v3
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v4
; GISEL-IEEE-NEXT: s_or_b64 s[4:5], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v2
; GISEL-IEEE-NEXT: s_cselect_b32 s8, s9, s8
@@ -4763,14 +4755,13 @@ define amdgpu_kernel void @elim_redun_check_v2_ult(ptr addrspace(1) %out, <2 x f
; GISEL-IEEE-NEXT: s_cselect_b32 s8, s4, s3
; GISEL-IEEE-NEXT: v_sqrt_f32_e32 v0, s8
; GISEL-IEEE-NEXT: s_cselect_b32 s7, 1, 0
+; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s8
; GISEL-IEEE-NEXT: v_readfirstlane_b32 s9, v0
; GISEL-IEEE-NEXT: s_add_i32 s10, s9, -1
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v1, s10
+; GISEL-IEEE-NEXT: v_fma_f32 v4, -s10, v0, v1
; GISEL-IEEE-NEXT: s_add_i32 s11, s9, 1
-; GISEL-IEEE-NEXT: v_fma_f32 v1, -v1, v0, s8
-; GISEL-IEEE-NEXT: v_mov_b32_e32 v4, s11
-; GISEL-IEEE-NEXT: v_fma_f32 v0, -v4, v0, s8
-; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GISEL-IEEE-NEXT: v_fma_f32 v0, -s11, v0, v1
+; GISEL-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v4
; GISEL-IEEE-NEXT: s_or_b64 s[4:5], vcc, vcc
; GISEL-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GISEL-IEEE-NEXT: s_cselect_b32 s9, s10, s9
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
index 5cd9738a8fa90..45d3c1d828fcf 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
@@ -877,13 +877,14 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64(double inreg %x) {
; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX6-GISEL-NEXT: v_bfrev_b32_e32 v1, 8
; GFX6-GISEL-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GFX6-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; GFX6-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; GFX6-GISEL-NEXT: s_lshl_b32 s3, s3, 8
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-GISEL-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; GFX6-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; GFX6-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX6-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX6-GISEL-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
+; GFX6-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX6-GISEL-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; GFX6-GISEL-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; GFX6-GISEL-NEXT: v_readfirstlane_b32 s1, v1
@@ -911,13 +912,14 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64(double inreg %x) {
; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX8-GISEL-NEXT: v_bfrev_b32_e32 v1, 8
; GFX8-GISEL-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX8-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; GFX8-GISEL-NEXT: s_lshl_b32 s3, s3, 8
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-GISEL-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; GFX8-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX8-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8-GISEL-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
+; GFX8-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX8-GISEL-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; GFX8-GISEL-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; GFX8-GISEL-NEXT: v_readfirstlane_b32 s1, v1
@@ -1012,13 +1014,14 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64_ninf(double inreg %x) {
; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX6-GISEL-NEXT: v_bfrev_b32_e32 v1, 8
; GFX6-GISEL-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GFX6-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; GFX6-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; GFX6-GISEL-NEXT: s_lshl_b32 s3, s3, 8
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-GISEL-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; GFX6-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; GFX6-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX6-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX6-GISEL-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
+; GFX6-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX6-GISEL-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; GFX6-GISEL-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; GFX6-GISEL-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[0:1]
@@ -1045,13 +1048,14 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64_ninf(double inreg %x) {
; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX8-GISEL-NEXT: v_bfrev_b32_e32 v1, 8
; GFX8-GISEL-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX8-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX8-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX8-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; GFX8-GISEL-NEXT: s_lshl_b32 s3, s3, 8
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-GISEL-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; GFX8-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX8-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8-GISEL-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
+; GFX8-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX8-GISEL-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; GFX8-GISEL-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; GFX8-GISEL-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll b/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll
index 56ae6abcefe77..080ac1c85496d 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll
@@ -282,13 +282,12 @@ define amdgpu_ps void @test_s_load_i16_divergent(ptr addrspace(4) inreg %in, i32
; GISEL-LABEL: test_s_load_i16_divergent:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
-; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v6, s1
-; GISEL-NEXT: v_mov_b32_e32 v5, s0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 1, v[0:1]
-; GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v5, v0
+; GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v6, v1, vcc_lo
+; GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GISEL-NEXT: global_load_i16 v0, v[0:1], off offset:32
; GISEL-NEXT: s_wait_loadcnt 0x0
; GISEL-NEXT: global_store_b32 v[3:4], v0, off
@@ -398,13 +397,12 @@ define amdgpu_ps void @test_s_load_u16_divergent(ptr addrspace(4) inreg %in, i32
; GISEL-LABEL: test_s_load_u16_divergent:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
-; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v6, s1
-; GISEL-NEXT: v_mov_b32_e32 v5, s0
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 1, v[0:1]
-; GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v5, v0
+; GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v6, v1, vcc_lo
+; GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GISEL-NEXT: global_load_u16 v0, v[0:1], off offset:32
; GISEL-NEXT: s_wait_loadcnt 0x0
; GISEL-NEXT: global_store_b32 v[3:4], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
index bde2505b35309..e419baf06cf8e 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
@@ -1938,11 +1938,10 @@ define amdgpu_ps float @global_load_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase, i3
;
; GFX12-GISEL-LABEL: global_load_i8_vgpr64_sgpr32:
; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc, v0, v2
-; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
+; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc, v0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s0, v1, vcc
; GFX12-GISEL-NEXT: global_load_u8 v0, v[0:1], off
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: ; return to shader part epilog
@@ -1994,11 +1993,10 @@ define amdgpu_ps float @global_load_i8_vgpr64_sgpr32_offset_4095(ptr addrspace(1
;
; GFX12-GISEL-LABEL: global_load_i8_vgpr64_sgpr32_offset_4095:
; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc, v0, v2
-; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
+; GFX12-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc, v0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s0, v1, vcc
; GFX12-GISEL-NEXT: global_load_u8 v0, v[0:1], off offset:4095
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: ; return to shader part epilog
@@ -2056,35 +2054,19 @@ define amdgpu_ps float @global_load_saddr_f32_natural_addressing(ptr addrspace(1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: global_load_saddr_f32_natural_addressing:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc, s2, v0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc
-; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: global_load_saddr_f32_natural_addressing:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc, v2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc
-; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-LABEL: global_load_saddr_f32_natural_addressing:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
+; GFX12-NEXT: v_add_co_u32 v0, vcc, s2, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
%gep = getelementptr inbounds float, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2237,35 +2219,19 @@ define amdgpu_ps float @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addr
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: global_load_f32_saddr_zext_vgpr_range_too_large:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc, s2, v0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc
-; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: global_load_f32_saddr_zext_vgpr_range_too_large:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc, v2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc
-; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-LABEL: global_load_f32_saddr_zext_vgpr_range_too_large:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
+; GFX12-NEXT: v_add_co_u32 v0, vcc, s2, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc
+; GFX12-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
%voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !1, !noundef !{}
%zext.offset = zext i32 %voffset to i64
%gep = getelementptr inbounds float, ptr addrspace(1) %sbase, i64 %zext.offset
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
index 96b3af6779e8c..d8d056bbeab15 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
@@ -7897,10 +7897,9 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v3, s5
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s4, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -7917,10 +7916,10 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX942-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX942-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -7937,10 +7936,8 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v3, s5
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s4
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -7959,10 +7956,9 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -7982,10 +7978,9 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -8008,10 +8003,9 @@ define <4 x float> @global_addr_64bit_lsr_iv(ptr addrspace(1) %arg) {
; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -8166,10 +8160,9 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX906-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX906-ISEL-NEXT: s_mov_b32 s5, 0
; GFX906-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v3, s5
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s4, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -8186,10 +8179,10 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX942-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
; GFX942-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX942-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX942-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off sc0
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -8206,10 +8199,8 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1012-ISEL-NEXT: ; %bb.2: ; %bb2
; GFX1012-ISEL-NEXT: s_mov_b32 s5, 0
; GFX1012-ISEL-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v3, s5
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s4
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -8228,10 +8219,9 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1100-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off glc
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -8251,10 +8241,9 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -8277,10 +8266,9 @@ define <4 x float> @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) %arg, pt
; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1310-ISEL-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -12413,10 +12401,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095(ptr addrspace(1)
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_4095:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc dlc
@@ -12528,10 +12514,9 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX906-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_4096:
; GFX906-ISEL: ; %bb.0:
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s17
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0
; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
@@ -12541,10 +12526,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX942-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_4096:
; GFX942-ISEL: ; %bb.0:
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0
; GFX942-ISEL-NEXT: s_nop 1
; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
@@ -12555,10 +12540,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_4096:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x1000, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
@@ -12568,12 +12551,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX1100-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_4096:
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x1000, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
@@ -12675,10 +12657,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4096(ptr addrspace(
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4096:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc
@@ -12790,10 +12770,9 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX906-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4097:
; GFX906-ISEL: ; %bb.0:
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s17
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffefff, v0
; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc
@@ -12803,10 +12782,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX942-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4097:
; GFX942-ISEL: ; %bb.0:
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffefff, v0
; GFX942-ISEL-NEXT: s_nop 1
; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
@@ -12817,10 +12796,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4097:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffefff, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc dlc
@@ -12830,12 +12807,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX1100-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg4097:
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffefff, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off glc
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
@@ -13040,10 +13016,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2048(ptr addrspace(1)
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_2048:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
@@ -13256,10 +13230,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2049(ptr addrspace(
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_neg2049:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff7ff, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc dlc
@@ -13371,10 +13343,9 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX906-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF:
; GFX906-ISEL: ; %bb.0:
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s17
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x7fffff, v0
; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc
@@ -13384,10 +13355,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX942-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF:
; GFX942-ISEL: ; %bb.0:
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x7fffff, v0
; GFX942-ISEL-NEXT: s_nop 1
; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
@@ -13398,10 +13369,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fffff, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc dlc
@@ -13411,12 +13380,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX1100-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF:
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fffff, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off glc
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
@@ -13519,10 +13487,9 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX906-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF:
; GFX906-ISEL: ; %bb.0:
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, s17
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xff800000, v0
; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
@@ -13532,10 +13499,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX942-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF:
; GFX942-ISEL: ; %bb.0:
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xff800000, v0
; GFX942-ISEL-NEXT: s_nop 1
; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
@@ -13546,10 +13513,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xff800000, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
@@ -13559,12 +13524,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX1100-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF:
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xff800000, v0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
@@ -13666,10 +13630,8 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095_gep_order(ptr add
; GFX1012-ISEL-LABEL: global_load_saddr_i8_zext_vgpr_offset_4095_gep_order:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s17
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v2, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, s4, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s17, 0, s4
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc
@@ -14844,10 +14806,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase,
; GFX906-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32:
; GFX906-ISEL: ; %bb.0:
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX906-ISEL-NEXT: s_mov_b32 s17, 0
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -14855,11 +14817,11 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase,
; GFX942-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32:
; GFX942-ISEL: ; %bb.0:
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-ISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX942-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off sc1
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -14867,10 +14829,9 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase,
; GFX1012-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX1012-ISEL-NEXT: s_mov_b32 s4, 0
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s16
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc dlc
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -14878,10 +14839,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase,
; GFX1100-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32:
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off glc
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -14890,10 +14851,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase,
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
-; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -14905,10 +14866,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32(ptr addrspace(1) %vbase,
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
-; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -14988,10 +14949,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32_offset_4095(ptr addrspace
; GFX906-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32_offset_4095:
; GFX906-ISEL: ; %bb.0:
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX906-ISEL-NEXT: s_mov_b32 s17, 0
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off offset:4095 glc
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -14999,11 +14960,11 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32_offset_4095(ptr addrspace
; GFX942-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32_offset_4095:
; GFX942-ISEL: ; %bb.0:
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-ISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX942-ISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-ISEL-NEXT: s_nop 1
-; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
; GFX942-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off offset:4095 sc0 sc1
; GFX942-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15011,10 +14972,9 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32_offset_4095(ptr addrspace
; GFX1012-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32_offset_4095:
; GFX1012-ISEL: ; %bb.0:
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX1012-ISEL-NEXT: s_mov_b32 s4, 0
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s16
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo
; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off glc dlc
@@ -15024,10 +14984,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32_offset_4095(ptr addrspace
; GFX1100-ISEL-LABEL: global_load_saddr_i8_vgpr64_sgpr32_offset_4095:
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1100-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 glc
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15036,10 +14996,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32_offset_4095(ptr addrspace
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
-; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -15051,10 +15011,10 @@ define <4 x float> @global_load_saddr_i8_vgpr64_sgpr32_offset_4095(ptr addrspace
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
-; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-ISEL-NEXT: s_mov_b32 s1, 0
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -15165,12 +15125,11 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: global_load_dword v0, v[0:1], off
; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v3, s17
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15191,12 +15150,10 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: global_load_dword v0, v[0:1], off
; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v3, s17
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s17, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15205,14 +15162,13 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
-; GFX1100-ISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15239,14 +15195,13 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -15764,12 +15719,11 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-ISEL-NEXT: global_load_dword v0, v[0:1], off
; GFX906-ISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX906-ISEL-NEXT: v_mov_b32_e32 v3, s17
+; GFX906-ISEL-NEXT: v_mov_b32_e32 v2, s17
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
-; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX906-ISEL-NEXT: v_add_co_u32_e32 v0, vcc, s16, v0
+; GFX906-ISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX906-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX906-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX906-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15790,12 +15744,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1012-ISEL-NEXT: global_load_dword v0, v[0:1], off
; GFX1012-ISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v2, s16
-; GFX1012-ISEL-NEXT: v_mov_b32_e32 v3, s17
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v3, v1, vcc_lo
+; GFX1012-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, s16, v0
+; GFX1012-ISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s17, v1, vcc_lo
; GFX1012-ISEL-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
; GFX1012-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1012-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15804,14 +15756,13 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1100-ISEL: ; %bb.0:
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1100-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
-; GFX1100-ISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1100-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1100-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1100-ISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1100-ISEL-NEXT: s_setpc_b64 s[30:31]
@@ -15838,14 +15789,13 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll
index 71565c90d0a72..e4e4b9acdd4cd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll
@@ -202,9 +202,8 @@ define amdgpu_kernel void @v_cvt_pk_i16_i32(ptr addrspace(1) %out, ptr addrspace
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll
index 86eef26e1b8ae..1d2b1d7547bb0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll
@@ -202,9 +202,8 @@ define amdgpu_kernel void @v_cvt_pk_u16_u32(ptr addrspace(1) %out, ptr addrspace
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
index 8bf907e24300f..acf64c6319d38 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
@@ -354,9 +354,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32(ptr addrspace(1) %out, ptr addr
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -728,9 +727,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo(ptr addrspace(1) %out,
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -928,9 +926,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_hi(ptr addrspace(1) %out,
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1129,9 +1126,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo_hi(ptr addrspace(1) %ou
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1335,9 +1331,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi(ptr addrsp
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
-; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; VI-GISEL-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-GISEL-NEXT: flat_load_dword v5, v[0:1] glc
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
index 5679dc7a5f6c1..2bca88d5996af 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
@@ -297,31 +297,22 @@ define amdgpu_ps void @global_load_async_to_lds_b64_saddr_no_scale_offset(ptr ad
; GFX1250-NEXT: global_load_async_to_lds_b64 v0, v[2:3], off offset:16 th:TH_LOAD_NT
; GFX1250-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: global_load_async_to_lds_b64_saddr_no_scale_offset:
-; GFX13-SDAG: ; %bb.0: ; %entry
-; GFX13-SDAG-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_lshlrev_b64_e32 v[1:2], 2, v[1:2]
-; GFX13-SDAG-NEXT: v_add_co_u32 v1, vcc_lo, s0, v1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_add_co_ci_u32_e64 v2, null, s1, v2, vcc_lo
-; GFX13-SDAG-NEXT: global_load_async_to_lds_b64 v0, v[1:2], off offset:16 th:TH_LOAD_NT
-; GFX13-SDAG-NEXT: s_endpgm
-;
-; GFX13-GISEL-LABEL: global_load_async_to_lds_b64_saddr_no_scale_offset:
-; GFX13-GISEL: ; %bb.0: ; %entry
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_ashrrev_i32 v2, 31, v1
-; GFX13-GISEL-NEXT: v_mov_b32_e32 v4, s1
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_lshlrev_b64_e32 v[1:2], 2, v[1:2]
-; GFX13-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v3, v1
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_add_co_ci_u32_e64 v2, null, v4, v2, vcc_lo
-; GFX13-GISEL-NEXT: global_load_async_to_lds_b64 v0, v[1:2], off offset:16 th:TH_LOAD_NT
-; GFX13-GISEL-NEXT: s_endpgm
+; GFX13-LABEL: global_load_async_to_lds_b64_saddr_no_scale_offset:
+; GFX13: ; %bb.0: ; %entry
+; GFX13-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_lshlrev_b64_e32 v[1:2], 2, v[1:2]
+; GFX13-NEXT: v_add_co_u32 v1, vcc_lo, s0, v1
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_add_co_ci_u32_e64 v2, null, s1, v2, vcc_lo
+; GFX13-NEXT: global_load_async_to_lds_b64 v0, v[1:2], off offset:16 th:TH_LOAD_NT
+; GFX13-NEXT: s_endpgm
entry:
%idxprom = sext i32 %idx to i64
%gep = getelementptr i32, ptr addrspace(1) %gaddr, i64 %idxprom
call void @llvm.amdgcn.global.load.async.to.lds.b64(ptr addrspace(1) %gep, ptr addrspace(3) %laddr, i32 16, i32 1)
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX13-GISEL: {{.*}}
+; GFX13-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index 541a7c73e8c07..1dce0b6e8687f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -1988,49 +1988,10 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
}
define amdgpu_ps i64 @v_icmp_i1_ne0_divergent(i32 %a) {
-; SDAG-GFX11-LABEL: v_icmp_i1_ne0_divergent:
-; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
-; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; SDAG-GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; SDAG-GFX11-NEXT: ; return to shader part epilog
-;
-; SDAG-VI-LABEL: v_icmp_i1_ne0_divergent:
-; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_readfirstlane_b32 s0, v0
-; SDAG-VI-NEXT: v_readfirstlane_b32 s1, v1
-; SDAG-VI-NEXT: ; return to shader part epilog
-;
-; SDAG-GFX9-LABEL: v_icmp_i1_ne0_divergent:
-; SDAG-GFX9: ; %bb.0:
-; SDAG-GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
-; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; SDAG-GFX9-NEXT: v_readfirstlane_b32 s1, v1
-; SDAG-GFX9-NEXT: ; return to shader part epilog
-;
-; GISEL-GFX11-LABEL: v_icmp_i1_ne0_divergent:
-; GISEL-GFX11: ; %bb.0:
-; GISEL-GFX11-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
-; GISEL-GFX11-NEXT: ; return to shader part epilog
-;
-; GISEL-VI-LABEL: v_icmp_i1_ne0_divergent:
-; GISEL-VI: ; %bb.0:
-; GISEL-VI-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
-; GISEL-VI-NEXT: ; return to shader part epilog
-;
-; GISEL-GFX9-LABEL: v_icmp_i1_ne0_divergent:
-; GISEL-GFX9: ; %bb.0:
-; GISEL-GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
-; GISEL-GFX9-NEXT: ; return to shader part epilog
+; GCN-LABEL: v_icmp_i1_ne0_divergent:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; GCN-NEXT: ; return to shader part epilog
%cond = icmp eq i32 %a, 0
%result = call i64 @llvm.amdgcn.icmp.i1(i1 %cond, i1 false, i32 33)
ret i64 %result
@@ -2070,5 +2031,4 @@ define amdgpu_ps void @test_intr_icmp_i32_invalid_cc(ptr addrspace(1) %out, i32
attributes #0 = { nounwind readnone convergent }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GCN: {{.*}}
; VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
index 5810d859f0cf4..c6e17542b89b5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
@@ -123,267 +123,106 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: v_lshlrev_b32_e32 v33, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v0, s0, v33
-; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:8208
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:8304
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v32, v34, a[96:127]
-; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:24592
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:24688
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: ds_read_b128 a[152:155], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[156:159], v0 offset:112
-; GISEL-NEXT: v_add_u32_e32 v33, s1, v33
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v3, 2.0
; GISEL-NEXT: ; iglp_opt mask(0x00000000)
-; GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v32, v34, a[0:31]
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:49264
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v32, v34, a[32:63]
-; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:32784
-; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:32816
-; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:32880
+; GISEL-NEXT: v_add_u32_e32 v1, s0, v0
+; GISEL-NEXT: v_add_u32_e32 v2, 0xc000, v1
+; GISEL-NEXT: ds_read_b128 a[0:3], v2 offset:32768
+; GISEL-NEXT: ds_read_b128 a[4:7], v2 offset:32784
+; GISEL-NEXT: ds_read_b128 a[8:11], v2 offset:32800
+; GISEL-NEXT: ds_read_b128 a[12:15], v2 offset:32816
+; GISEL-NEXT: ds_read_b128 a[16:19], v2 offset:32832
+; GISEL-NEXT: ds_read_b128 a[20:23], v2 offset:32848
+; GISEL-NEXT: ds_read_b128 a[24:27], v2 offset:32864
+; GISEL-NEXT: ds_read_b128 a[28:31], v2 offset:32880
+; GISEL-NEXT: v_mov_b32_e32 v2, 1.0
+; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:49152
+; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:49168
+; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:49184
+; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:49200
+; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:49216
+; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:49232
+; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:49248
+; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:49264
+; GISEL-NEXT: s_waitcnt lgkmcnt(8)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; GISEL-NEXT: ds_read_b128 a[32:35], v1
+; GISEL-NEXT: ds_read_b128 a[36:39], v1 offset:16
+; GISEL-NEXT: ds_read_b128 a[120:123], v1 offset:24672
+; GISEL-NEXT: ds_read_b128 a[124:127], v1 offset:24688
+; GISEL-NEXT: v_add_u32_e32 v0, s1, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159]
+; GISEL-NEXT: ds_read_b128 a[40:43], v1 offset:32
+; GISEL-NEXT: ds_read_b128 a[44:47], v1 offset:48
+; GISEL-NEXT: ds_read_b128 a[48:51], v1 offset:64
+; GISEL-NEXT: ds_read_b128 a[52:55], v1 offset:80
+; GISEL-NEXT: ds_read_b128 a[56:59], v1 offset:96
+; GISEL-NEXT: ds_read_b128 a[60:63], v1 offset:112
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v32, v34, a[64:95]
-; GISEL-NEXT: ds_read_b128 a[128:131], v0
-; GISEL-NEXT: ds_read_b128 a[132:135], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[136:139], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[140:143], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[144:147], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[148:151], v0 offset:80
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:8192
+; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:8208
+; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:8224
+; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:8240
+; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:8256
+; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:8272
+; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:8288
+; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:8304
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v32, v34, a[128:159]
-; GISEL-NEXT: v_mov_b32_e32 v32, s1
-; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
+; GISEL-NEXT: ds_read_b128 a[96:99], v1 offset:24576
+; GISEL-NEXT: ds_read_b128 a[100:103], v1 offset:24592
+; GISEL-NEXT: ds_read_b128 a[104:107], v1 offset:24608
+; GISEL-NEXT: ds_read_b128 a[108:111], v1 offset:24624
+; GISEL-NEXT: ds_read_b128 a[112:115], v1 offset:24640
+; GISEL-NEXT: ds_read_b128 a[116:119], v1 offset:24656
+; GISEL-NEXT: v_mov_b32_e32 v1, s1
; GISEL-NEXT: s_nop 1
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ds_write_b128 v33, v[0:3]
-; GISEL-NEXT: ds_write_b128 v33, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v33, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v33, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v33, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v33, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v33, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v33, v[28:31] offset:112
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: ds_write_b128 v0, a[32:35]
+; GISEL-NEXT: ds_write_b128 v0, a[36:39] offset:16
+; GISEL-NEXT: ds_write_b128 v0, a[40:43] offset:32
+; GISEL-NEXT: ds_write_b128 v0, a[44:47] offset:48
+; GISEL-NEXT: ds_write_b128 v0, a[48:51] offset:64
+; GISEL-NEXT: ds_write_b128 v0, a[52:55] offset:80
+; GISEL-NEXT: ds_write_b128 v0, a[56:59] offset:96
+; GISEL-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; GISEL-NEXT: s_waitcnt lgkmcnt(8)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127]
+; GISEL-NEXT: ds_write_b128 v1, a[128:131] offset:24576
+; GISEL-NEXT: ds_write_b128 v1, a[132:135] offset:24592
+; GISEL-NEXT: ds_write_b128 v1, a[136:139] offset:24608
+; GISEL-NEXT: ds_write_b128 v1, a[64:67] offset:8192
+; GISEL-NEXT: ds_write_b128 v1, a[68:71] offset:8208
+; GISEL-NEXT: ds_write_b128 v1, a[72:75] offset:8224
+; GISEL-NEXT: ds_write_b128 v1, a[76:79] offset:8240
+; GISEL-NEXT: ds_write_b128 v1, a[80:83] offset:8256
+; GISEL-NEXT: ds_write_b128 v1, a[84:87] offset:8272
+; GISEL-NEXT: ds_write_b128 v1, a[88:91] offset:8288
+; GISEL-NEXT: ds_write_b128 v1, a[92:95] offset:8304
+; GISEL-NEXT: ds_write_b128 v1, a[140:143] offset:24624
+; GISEL-NEXT: ds_write_b128 v1, a[144:147] offset:24640
+; GISEL-NEXT: ds_write_b128 v1, a[148:151] offset:24656
+; GISEL-NEXT: ds_write_b128 v1, a[152:155] offset:24672
+; GISEL-NEXT: ds_write_b128 v1, a[156:159] offset:24688
+; GISEL-NEXT: ds_write_b128 v1, a[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v1, a[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v1, a[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v1, a[96:99] offset:16384
+; GISEL-NEXT: ds_write_b128 v1, a[100:103] offset:16400
+; GISEL-NEXT: ds_write_b128 v1, a[104:107] offset:16416
+; GISEL-NEXT: ds_write_b128 v1, a[108:111] offset:16432
+; GISEL-NEXT: ds_write_b128 v1, a[112:115] offset:16448
+; GISEL-NEXT: ds_write_b128 v1, a[116:119] offset:16464
+; GISEL-NEXT: ds_write_b128 v1, a[120:123] offset:16480
+; GISEL-NEXT: ds_write_b128 v1, a[124:127] offset:16496
+; GISEL-NEXT: ds_write_b128 v1, a[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v1, a[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v1, a[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v1, a[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v1, a[28:31] offset:32880
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.iglp.opt(i32 0)
@@ -527,265 +366,105 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v1, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v2, 2.0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v0, s0, v32
-; GISEL-NEXT: ds_read_b128 a[0:3], v0
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:80
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; GISEL-NEXT: v_add_u32_e32 v3, s0, v0
+; GISEL-NEXT: ds_read_b128 a[0:3], v3
+; GISEL-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v3 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v3 offset:112
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:8208
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:8304
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: v_add_u32_e32 v32, s1, v32
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
+; GISEL-NEXT: ds_read_b128 a[32:35], v3 offset:8192
+; GISEL-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; GISEL-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; GISEL-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; GISEL-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; GISEL-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; GISEL-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; GISEL-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; GISEL-NEXT: v_add_u32_e32 v4, 0xc000, v3
+; GISEL-NEXT: v_add_u32_e32 v0, s1, v0
; GISEL-NEXT: ; iglp_opt mask(0x00000001)
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63]
-; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:24592
-; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v1, v2, a[32:63]
+; GISEL-NEXT: ds_read_b128 a[64:67], v3 offset:24576
+; GISEL-NEXT: ds_read_b128 a[68:71], v3 offset:24592
+; GISEL-NEXT: ds_read_b128 a[72:75], v3 offset:24608
+; GISEL-NEXT: ds_read_b128 a[76:79], v3 offset:24624
+; GISEL-NEXT: ds_read_b128 a[80:83], v3 offset:24640
+; GISEL-NEXT: ds_read_b128 a[84:87], v3 offset:24656
+; GISEL-NEXT: ds_read_b128 a[88:91], v3 offset:24672
+; GISEL-NEXT: ds_read_b128 a[92:95], v3 offset:24688
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95]
-; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v1, v2, a[64:95]
+; GISEL-NEXT: ds_read_b128 a[96:99], v3 offset:49152
+; GISEL-NEXT: ds_read_b128 a[100:103], v3 offset:49168
+; GISEL-NEXT: ds_read_b128 a[104:107], v3 offset:49184
+; GISEL-NEXT: ds_read_b128 a[108:111], v3 offset:49200
+; GISEL-NEXT: ds_read_b128 a[112:115], v3 offset:49216
+; GISEL-NEXT: ds_read_b128 a[116:119], v3 offset:49232
+; GISEL-NEXT: ds_read_b128 a[120:123], v3 offset:49248
+; GISEL-NEXT: ds_read_b128 a[124:127], v3 offset:49264
+; GISEL-NEXT: v_mov_b32_e32 v3, s1
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127]
-; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
-; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
-; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v1, v2, a[96:127]
+; GISEL-NEXT: ds_read_b128 a[128:131], v4 offset:32768
+; GISEL-NEXT: ds_read_b128 a[132:135], v4 offset:32784
+; GISEL-NEXT: ds_read_b128 a[136:139], v4 offset:32800
+; GISEL-NEXT: ds_read_b128 a[140:143], v4 offset:32816
+; GISEL-NEXT: ds_read_b128 a[144:147], v4 offset:32832
+; GISEL-NEXT: ds_read_b128 a[148:151], v4 offset:32848
+; GISEL-NEXT: ds_read_b128 a[152:155], v4 offset:32864
+; GISEL-NEXT: ds_read_b128 a[156:159], v4 offset:32880
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v32, v[0:3]
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: v_mov_b32_e32 v32, s1
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v1, v2, a[128:159]
+; GISEL-NEXT: ds_write_b128 v0, a[0:3]
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; GISEL-NEXT: ds_write_b128 v3, a[32:35] offset:8192
+; GISEL-NEXT: ds_write_b128 v3, a[36:39] offset:8208
+; GISEL-NEXT: ds_write_b128 v3, a[40:43] offset:8224
+; GISEL-NEXT: ds_write_b128 v3, a[44:47] offset:8240
+; GISEL-NEXT: ds_write_b128 v3, a[48:51] offset:8256
+; GISEL-NEXT: ds_write_b128 v3, a[52:55] offset:8272
+; GISEL-NEXT: ds_write_b128 v3, a[56:59] offset:8288
+; GISEL-NEXT: ds_write_b128 v3, a[60:63] offset:8304
+; GISEL-NEXT: ds_write_b128 v3, a[64:67] offset:16384
+; GISEL-NEXT: ds_write_b128 v3, a[68:71] offset:16400
+; GISEL-NEXT: ds_write_b128 v3, a[72:75] offset:16416
+; GISEL-NEXT: ds_write_b128 v3, a[76:79] offset:16432
+; GISEL-NEXT: ds_write_b128 v3, a[80:83] offset:16448
+; GISEL-NEXT: ds_write_b128 v3, a[84:87] offset:16464
+; GISEL-NEXT: ds_write_b128 v3, a[88:91] offset:16480
+; GISEL-NEXT: ds_write_b128 v3, a[92:95] offset:16496
+; GISEL-NEXT: ds_write_b128 v3, a[96:99] offset:24576
+; GISEL-NEXT: ds_write_b128 v3, a[100:103] offset:24592
+; GISEL-NEXT: ds_write_b128 v3, a[104:107] offset:24608
+; GISEL-NEXT: ds_write_b128 v3, a[108:111] offset:24624
+; GISEL-NEXT: ds_write_b128 v3, a[112:115] offset:24640
+; GISEL-NEXT: ds_write_b128 v3, a[116:119] offset:24656
+; GISEL-NEXT: ds_write_b128 v3, a[120:123] offset:24672
+; GISEL-NEXT: ds_write_b128 v3, a[124:127] offset:24688
+; GISEL-NEXT: ds_write_b128 v3, a[128:131] offset:32768
+; GISEL-NEXT: ds_write_b128 v3, a[132:135] offset:32784
+; GISEL-NEXT: ds_write_b128 v3, a[136:139] offset:32800
+; GISEL-NEXT: ds_write_b128 v3, a[140:143] offset:32816
+; GISEL-NEXT: ds_write_b128 v3, a[144:147] offset:32832
+; GISEL-NEXT: ds_write_b128 v3, a[148:151] offset:32848
+; GISEL-NEXT: ds_write_b128 v3, a[152:155] offset:32864
+; GISEL-NEXT: ds_write_b128 v3, a[156:159] offset:32880
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.iglp.opt(i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 3f931eaad33e3..ce52285f4c771 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -1073,7 +1073,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1013-GISEL-NEXT: s_clause 0x1
; GFX1013-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1013-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
@@ -1084,10 +1084,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v10, 0x40e00000
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v11, 0x41000000
; GFX1013-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; GFX1013-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX1013-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s7, 0, s4
; GFX1013-GISEL-NEXT: flat_load_dword v2, v[0:1]
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, 0xb36211c7
; GFX1013-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -1129,7 +1127,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1030-GISEL-NEXT: s_clause 0x1
; GFX1030-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1030-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
@@ -1140,10 +1138,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v10, 0x40e00000
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v11, 0x41000000
; GFX1030-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX1030-GISEL-NEXT: flat_load_dword v2, v[0:1]
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, 0xb36211c7
; GFX1030-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -1186,19 +1182,17 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: v_bfrev_b32_e32 v10, 4.0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, 2.0 :: v_dual_mov_b32 v5, 0x40a00000
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v9, 0xb36211c7 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-GISEL-NEXT: v_dual_mov_b32 v4, 4.0 :: v_dual_mov_b32 v7, 0x40e00000
; GFX11-GISEL-NEXT: v_mov_b32_e32 v8, 0x41000000
; GFX11-GISEL-NEXT: v_mov_b32_e32 v6, 0x40c00000
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v9, 0xb36211c7
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v5, 0x40a00000
; GFX11-GISEL-NEXT: v_mov_b32_e32 v3, 0x40400000
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 2.0
+; GFX11-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX11-GISEL-NEXT: flat_load_b32 v11, v[0:1]
; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 1.0 :: v_dual_mov_b32 v0, 0
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1240,19 +1234,17 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
; GFX12-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: v_bfrev_b32_e32 v10, 4.0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 2.0 :: v_dual_mov_b32 v5, 0x40a00000
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v9, 0xb36211c7 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX12-GISEL-NEXT: v_dual_mov_b32 v4, 4.0 :: v_dual_mov_b32 v7, 0x40e00000
; GFX12-GISEL-NEXT: v_mov_b32_e32 v8, 0x41000000
; GFX12-GISEL-NEXT: v_mov_b32_e32 v6, 0x40c00000
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v9, 0xb36211c7
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v5, 0x40a00000
; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, 0x40400000
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 2.0
+; GFX12-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX12-GISEL-NEXT: flat_load_b32 v11, v[0:1]
; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 1.0 :: v_dual_mov_b32 v0, 0
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1308,7 +1300,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-GISEL-NEXT: s_clause 0x1
; GFX1013-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1013-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
@@ -1316,10 +1308,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, 0x46004500
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1013-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; GFX1013-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX1013-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s7, 0, s4
; GFX1013-GISEL-NEXT: flat_load_dword v2, v[0:1]
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, 0xb36211c6
; GFX1013-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
@@ -1358,7 +1348,7 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1030-GISEL-NEXT: s_clause 0x1
; GFX1030-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1030-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
@@ -1366,10 +1356,8 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, 0x46004500
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1030-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
+; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX1030-GISEL-NEXT: flat_load_dword v2, v[0:1]
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, 0xb36211c6
; GFX1030-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
index 194277de08dcb..a954f9f2ab85c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -27,24 +27,24 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
-; CI-SDAG-LABEL: is_private_vgpr:
-; CI-SDAG: ; %bb.0:
-; CI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-SDAG-NEXT: s_load_dword s2, s[8:9], 0x32
-; CI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; CI-SDAG-NEXT: s_add_i32 s12, s12, s17
-; CI-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; CI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; CI-SDAG-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; CI-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
-; CI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; CI-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
-; CI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CI-SDAG-NEXT: flat_store_dword v[0:1], v0
-; CI-SDAG-NEXT: s_endpgm
+; CI-LABEL: is_private_vgpr:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dword s2, s[8:9], 0x32
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: s_add_i32 s12, s12, s17
+; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
+; CI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CI-NEXT: flat_store_dword v[0:1], v0
+; CI-NEXT: s_endpgm
;
; GFX9-LABEL: is_private_vgpr:
; GFX9: ; %bb.0:
@@ -77,26 +77,6 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
; GFX1250-NEXT: global_store_b32 v[0:1], v0, off
; GFX1250-NEXT: s_endpgm
;
-; CI-GISEL-LABEL: is_private_vgpr:
-; CI-GISEL: ; %bb.0:
-; CI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-GISEL-NEXT: s_load_dword s2, s[8:9], 0x32
-; CI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; CI-GISEL-NEXT: s_add_i32 s12, s12, s17
-; CI-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; CI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; CI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; CI-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; CI-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
-; CI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; CI-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
-; CI-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CI-GISEL-NEXT: flat_store_dword v[0:1], v0
-; CI-GISEL-NEXT: s_endpgm
-;
; GFX10-LABEL: is_private_vgpr:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
@@ -300,7 +280,6 @@ bb1:
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CI: {{.*}}
; GFX10-GISEL: {{.*}}
; GFX11-GISEL: {{.*}}
; SI-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
index f41bb992cacab..947e9d2d9abf3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
@@ -60,24 +60,24 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
-; CI-SDAG-LABEL: is_local_vgpr:
-; CI-SDAG: ; %bb.0:
-; CI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-SDAG-NEXT: s_load_dword s2, s[8:9], 0x33
-; CI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; CI-SDAG-NEXT: s_add_i32 s12, s12, s17
-; CI-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; CI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; CI-SDAG-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; CI-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
-; CI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; CI-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
-; CI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CI-SDAG-NEXT: flat_store_dword v[0:1], v0
-; CI-SDAG-NEXT: s_endpgm
+; CI-LABEL: is_local_vgpr:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dword s2, s[8:9], 0x33
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: s_add_i32 s12, s12, s17
+; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
+; CI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CI-NEXT: flat_store_dword v[0:1], v0
+; CI-NEXT: s_endpgm
;
; GFX9-LABEL: is_local_vgpr:
; GFX9: ; %bb.0:
@@ -109,26 +109,6 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
; GFX1250-NEXT: global_store_b32 v[0:1], v0, off
; GFX1250-NEXT: s_endpgm
;
-; CI-GISEL-LABEL: is_local_vgpr:
-; CI-GISEL: ; %bb.0:
-; CI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-GISEL-NEXT: s_load_dword s2, s[8:9], 0x33
-; CI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; CI-GISEL-NEXT: s_add_i32 s12, s12, s17
-; CI-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; CI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; CI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; CI-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; CI-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CI-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
-; CI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; CI-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
-; CI-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CI-GISEL-NEXT: flat_store_dword v[0:1], v0
-; CI-GISEL-NEXT: s_endpgm
-;
; GFX10-LABEL: is_local_vgpr:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
@@ -365,7 +345,6 @@ bb1:
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CI: {{.*}}
; GFX10-GISEL: {{.*}}
; GFX11-GISEL: {{.*}}
; SI-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
index 5abfd34619de6..21ccf4cc0492b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
@@ -1828,16 +1828,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 0x3ff00000
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 1.0
; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-GISEL-NEXT: s_nop 15
@@ -1867,15 +1862,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], 1.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 1.0
; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX942-GISEL-NEXT: s_nop 15
@@ -1905,16 +1896,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, 0x3ff00000
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: s_nop 1
-; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 1.0
; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-VGPR-GISEL-NEXT: s_nop 15
@@ -1944,15 +1930,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 1.0
; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX942-VGPR-GISEL-NEXT: s_nop 15
@@ -1989,16 +1971,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 0xbff00000
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1.0
; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-GISEL-NEXT: s_nop 15
@@ -2028,15 +2005,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], -1.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1.0
; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX942-GISEL-NEXT: s_nop 15
@@ -2066,16 +2039,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, 0xbff00000
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: s_nop 1
-; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1.0
; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-VGPR-GISEL-NEXT: s_nop 15
@@ -2105,15 +2073,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], -1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1.0
; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX942-VGPR-GISEL-NEXT: s_nop 15
@@ -2150,16 +2114,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 64
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 64
; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-GISEL-NEXT: s_nop 15
@@ -2189,15 +2148,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], 64
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 64
; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX942-GISEL-NEXT: s_nop 15
@@ -2227,16 +2182,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 64
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-GISEL-NEXT: s_nop 1
-; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 64
; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX90A-VGPR-GISEL-NEXT: s_nop 15
@@ -2266,15 +2216,11 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], 64
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 64
; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX942-VGPR-GISEL-NEXT: s_nop 15
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index ce660cd4bb6c3..e6f6e3bca7177 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -147,9 +147,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; GISEL-AGPR: ; %bb.0:
; GISEL-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[20:21], 0
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[22:23], 16
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[24:25], 32
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[12:13], 0
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[14:15], 16
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], 32
; GISEL-AGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
@@ -171,50 +171,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], 48
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], 48
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; GISEL-AGPR-NEXT: s_nop 10
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v0, a16
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v1, a17
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v2, a18
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v3, a19
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v4, a20
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v5, a21
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v6, a22
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v7, a23
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v8, a24
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v9, a25
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v10, a26
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v11, a27
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v12, a28
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v13, a29
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v14, a30
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v15, a31
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-AGPR-NEXT: s_nop 8
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[22:23], v[4:7], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[24:25], v[8:11], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[26:27], v[12:15], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_nop 0
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_nop 0
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_endpgm
;
@@ -470,9 +454,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; GISEL-AGPR: ; %bb.0:
; GISEL-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[20:21], 0
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[22:23], 16
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[24:25], 32
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[12:13], 0
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[14:15], 16
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], 32
; GISEL-AGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
@@ -494,50 +478,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], 48
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], 48
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; GISEL-AGPR-NEXT: s_nop 10
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v0, a16
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v1, a17
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v2, a18
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v3, a19
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v4, a20
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v5, a21
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v6, a22
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v7, a23
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v8, a24
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v9, a25
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v10, a26
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v11, a27
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v12, a28
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v13, a29
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v14, a30
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v15, a31
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-AGPR-NEXT: s_nop 8
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[22:23], v[4:7], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[24:25], v[8:11], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[26:27], v[12:15], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_nop 0
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_nop 0
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 49d4662300f76..6d9ecfb4551f6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -187,7 +187,6 @@ define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -209,28 +208,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a15, s15
; GFX90A-GISEL-NEXT: s_nop 1
; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
; GFX90A-GISEL-NEXT: s_endpgm
;
; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_32x32x8i8:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 413c2973a3931..28f15308f5928 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -4950,26 +4950,10 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 64
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, 64 cbsz:1 abid:2 blgp:3
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: s_nop 8
@@ -4998,26 +4982,10 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 64
; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, 64 cbsz:1 abid:2 blgp:3
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: s_nop 8
@@ -5046,26 +5014,10 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 64
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, 64 cbsz:1 abid:2 blgp:3
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-GISEL-NEXT: s_nop 8
@@ -5370,18 +5322,14 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
;
; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: s_nop 3
; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-GISEL-NEXT: s_endpgm
;
@@ -5400,18 +5348,14 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
;
; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: s_nop 3
; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-GISEL-NEXT: s_endpgm
;
@@ -5430,18 +5374,14 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
;
; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, 2
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
@@ -7001,18 +6941,14 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
;
; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v0, v1, 1.0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: s_nop 3
; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-GISEL-NEXT: s_endpgm
;
@@ -7031,18 +6967,14 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
;
; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, 1.0
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: s_nop 2
; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-GISEL-NEXT: s_endpgm
;
@@ -7061,18 +6993,14 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
;
; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, 1.0
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
@@ -7267,26 +7195,10 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, 1.0
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: s_nop 8
@@ -7315,26 +7227,10 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, 1.0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: s_nop 7
@@ -7363,26 +7259,10 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, 1.0
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-GISEL-NEXT: s_nop 7
@@ -7599,26 +7479,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0x3c003c00
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, v16
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v18, 0x40004000
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v19, v18
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], 1.0
; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: s_nop 15
@@ -7652,26 +7516,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0x3c003c00
; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, v16
; GFX942-GISEL-NEXT: v_mov_b32_e32 v18, 0x40004000
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v19, v18
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], 1.0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: s_nop 8
@@ -7704,26 +7552,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0x3c003c00
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, v16
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v18, 0x40004000
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v19, v18
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v15, v0
; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], 1.0
; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-GISEL-NEXT: s_nop 8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 749f9e5ae5ef2..3d92b6b2743fe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -7888,29 +7888,13 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; AGPR-GISEL-NEXT: global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
; AGPR-GISEL-NEXT: s_nop 2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[16:17], a[0:3], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[18:19], a[4:7], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[20:21], v[8:11], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[20:21], a[8:11], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[22:23], a[12:15], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
; AGPR-GISEL-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
@@ -8180,29 +8164,13 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; AGPR-GISEL-NEXT: global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
; AGPR-GISEL-NEXT: s_nop 2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[16:17], a[0:3], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[18:19], a[4:7], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[20:21], v[8:11], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[20:21], a[8:11], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[22:23], a[12:15], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
; AGPR-GISEL-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 0, i32 2, i32 0, i32 25, i32 0, i32 42)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
index 763b618ae8335..c96bba933e7ad 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
@@ -1,8 +1,8 @@
-; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT,VI-OPT-SDAG,PREGFX10,PREGFX10-OPT %s
+; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT,PREGFX10,PREGFX10-OPT %s
; RUN: llc -O0 -mtriple=amdgpu8.02 -mattr=-flat-for-global -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-NOOPT,PREGFX10,PREGFX10-NOOPT %s
-; RUN: llc -mtriple=amdgpu10.10 -mattr=-flat-for-global -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT,VI-OPT-SDAG %s
-; RUN: llc -mtriple=amdgpu11.00 -mattr=-flat-for-global -amdgpu-enable-vopd=0 -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT,VI-OPT-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-flat-for-global -amdgpu-enable-vopd=0 -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT,VI-OPT-GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -mattr=-flat-for-global -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT %s
+; RUN: llc -mtriple=amdgpu11.00 -mattr=-flat-for-global -amdgpu-enable-vopd=0 -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-flat-for-global -amdgpu-enable-vopd=0 -show-mc-encoding < %s | FileCheck -check-prefixes=VI,VI-OPT %s
; FIXME: The register allocator / scheduler should be able to avoid these hazards.
@@ -79,12 +79,8 @@ define amdgpu_kernel void @mov_dpp64_test(ptr addrspace(1) %out, i64 %in1) {
}
; VI-LABEL: {{^}}mov_dpp64_imm_test:
-; VI-OPT-SDAG-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x3afaedd9
-; VI-OPT-SDAG-DAG: s_movk_i32 s[[SOLD_HI:[0-9]+]], 0x7047
-; VI-OPT-SDAG-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], s[[SOLD_LO]]
-; VI-OPT-SDAG-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], s[[SOLD_HI]]
-; VI-OPT-GISEL-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x3afaedd9
-; VI-OPT-GISEL-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
+; VI-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x3afaedd9
+; VI-OPT-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
; VI-OPT-DAG: v_mov_b32_dpp v[[OLD_LO]], v[[OLD_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; VI-OPT-DAG: v_mov_b32_dpp v[[OLD_HI]], v[[OLD_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; VI-NOOPT-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 916c0a2e87e9f..8b73b6e07b984 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -76,22 +76,21 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
; VARIANT2-GISEL: ; %bb.0: ; %entry
; VARIANT2-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VARIANT2-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VARIANT2-GISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
+; VARIANT2-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VARIANT2-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VARIANT2-GISEL-NEXT: global_store_dword v4, v0, s[0:1]
+; VARIANT2-GISEL-NEXT: global_store_dword v2, v0, s[0:1]
; VARIANT2-GISEL-NEXT: s_add_i32 s2, s2, -1
; VARIANT2-GISEL-NEXT: v_sub_u32_e32 v0, s2, v0
; VARIANT2-GISEL-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VARIANT2-GISEL-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
-; VARIANT2-GISEL-NEXT: v_mov_b32_e32 v2, s0
; VARIANT2-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; VARIANT2-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; VARIANT2-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; VARIANT2-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
; VARIANT2-GISEL-NEXT: s_waitcnt vmcnt(0)
; VARIANT2-GISEL-NEXT: s_barrier
; VARIANT2-GISEL-NEXT: global_load_dword v0, v[0:1], off
; VARIANT2-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VARIANT2-GISEL-NEXT: global_store_dword v4, v0, s[0:1]
+; VARIANT2-GISEL-NEXT: global_store_dword v2, v0, s[0:1]
; VARIANT2-GISEL-NEXT: s_endpgm
;
; VARIANT3-LABEL: test_barrier:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 96edd7876e981..7bb3990c32cfe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -964,265 +964,104 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v1, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v2, 2.0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v0, s0, v32
-; GISEL-NEXT: ds_read_b128 a[96:99], v0
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:80
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:112
-; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:49264
-; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
-; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
-; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
+; GISEL-NEXT: v_add_u32_e32 v3, s0, v0
+; GISEL-NEXT: v_add_u32_e32 v4, 0xc000, v3
+; GISEL-NEXT: ds_read_b128 a[0:3], v3
+; GISEL-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v3 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; GISEL-NEXT: ds_read_b128 a[32:35], v3 offset:8192
+; GISEL-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; GISEL-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; GISEL-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; GISEL-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; GISEL-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; GISEL-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; GISEL-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; GISEL-NEXT: ds_read_b128 a[64:67], v3 offset:24576
+; GISEL-NEXT: ds_read_b128 a[68:71], v3 offset:24592
+; GISEL-NEXT: ds_read_b128 a[72:75], v3 offset:24608
+; GISEL-NEXT: ds_read_b128 a[76:79], v3 offset:24624
+; GISEL-NEXT: ds_read_b128 a[80:83], v3 offset:24640
+; GISEL-NEXT: ds_read_b128 a[84:87], v3 offset:24656
+; GISEL-NEXT: ds_read_b128 a[88:91], v3 offset:24672
+; GISEL-NEXT: ds_read_b128 a[92:95], v3 offset:24688
+; GISEL-NEXT: ds_read_b128 a[96:99], v3 offset:49152
+; GISEL-NEXT: ds_read_b128 a[100:103], v3 offset:49168
+; GISEL-NEXT: ds_read_b128 a[104:107], v3 offset:49184
+; GISEL-NEXT: ds_read_b128 a[108:111], v3 offset:49200
+; GISEL-NEXT: ds_read_b128 a[112:115], v3 offset:49216
+; GISEL-NEXT: ds_read_b128 a[116:119], v3 offset:49232
+; GISEL-NEXT: ds_read_b128 a[120:123], v3 offset:49248
+; GISEL-NEXT: ds_read_b128 a[124:127], v3 offset:49264
+; GISEL-NEXT: ds_read_b128 a[128:131], v4 offset:32768
+; GISEL-NEXT: ds_read_b128 a[132:135], v4 offset:32784
+; GISEL-NEXT: ds_read_b128 a[136:139], v4 offset:32800
+; GISEL-NEXT: ds_read_b128 a[140:143], v4 offset:32816
+; GISEL-NEXT: ds_read_b128 a[144:147], v4 offset:32832
+; GISEL-NEXT: ds_read_b128 a[148:151], v4 offset:32848
+; GISEL-NEXT: ds_read_b128 a[152:155], v4 offset:32864
+; GISEL-NEXT: ds_read_b128 a[156:159], v4 offset:32880
; GISEL-NEXT: s_waitcnt lgkmcnt(14)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127]
-; GISEL-NEXT: v_add_u32_e32 v32, s1, v32
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
+; GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-NEXT: v_add_u32_e32 v0, s1, v0
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
-; GISEL-NEXT: s_nop 15
-; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63]
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
; GISEL-NEXT: s_waitcnt lgkmcnt(8)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95]
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v1, v2, a[96:127]
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v1, v2, a[64:95]
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v1, v2, a[32:63]
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
-; GISEL-NEXT: ds_write_b128 v32, v[0:3]
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: v_mov_b32_e32 v32, s1
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v1, v2, a[128:159]
+; GISEL-NEXT: s_nop 10
+; GISEL-NEXT: ds_write_b128 v0, a[0:3]
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; GISEL-NEXT: ds_write_b128 v3, a[32:35] offset:8192
+; GISEL-NEXT: ds_write_b128 v3, a[36:39] offset:8208
+; GISEL-NEXT: ds_write_b128 v3, a[40:43] offset:8224
+; GISEL-NEXT: ds_write_b128 v3, a[44:47] offset:8240
+; GISEL-NEXT: ds_write_b128 v3, a[48:51] offset:8256
+; GISEL-NEXT: ds_write_b128 v3, a[52:55] offset:8272
+; GISEL-NEXT: ds_write_b128 v3, a[56:59] offset:8288
+; GISEL-NEXT: ds_write_b128 v3, a[60:63] offset:8304
+; GISEL-NEXT: ds_write_b128 v3, a[64:67] offset:16384
+; GISEL-NEXT: ds_write_b128 v3, a[68:71] offset:16400
+; GISEL-NEXT: ds_write_b128 v3, a[72:75] offset:16416
+; GISEL-NEXT: ds_write_b128 v3, a[76:79] offset:16432
+; GISEL-NEXT: ds_write_b128 v3, a[80:83] offset:16448
+; GISEL-NEXT: ds_write_b128 v3, a[84:87] offset:16464
+; GISEL-NEXT: ds_write_b128 v3, a[88:91] offset:16480
+; GISEL-NEXT: ds_write_b128 v3, a[92:95] offset:16496
+; GISEL-NEXT: ds_write_b128 v3, a[96:99] offset:24576
+; GISEL-NEXT: ds_write_b128 v3, a[100:103] offset:24592
+; GISEL-NEXT: ds_write_b128 v3, a[104:107] offset:24608
+; GISEL-NEXT: ds_write_b128 v3, a[108:111] offset:24624
+; GISEL-NEXT: ds_write_b128 v3, a[112:115] offset:24640
+; GISEL-NEXT: ds_write_b128 v3, a[116:119] offset:24656
+; GISEL-NEXT: ds_write_b128 v3, a[120:123] offset:24672
+; GISEL-NEXT: ds_write_b128 v3, a[124:127] offset:24688
+; GISEL-NEXT: ds_write_b128 v3, a[128:131] offset:32768
+; GISEL-NEXT: ds_write_b128 v3, a[132:135] offset:32784
+; GISEL-NEXT: ds_write_b128 v3, a[136:139] offset:32800
+; GISEL-NEXT: ds_write_b128 v3, a[140:143] offset:32816
+; GISEL-NEXT: ds_write_b128 v3, a[144:147] offset:32832
+; GISEL-NEXT: ds_write_b128 v3, a[148:151] offset:32848
+; GISEL-NEXT: ds_write_b128 v3, a[152:155] offset:32864
+; GISEL-NEXT: ds_write_b128 v3, a[156:159] offset:32880
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
; GISEL-NEXT: s_endpgm
@@ -1503,287 +1342,127 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: v_mov_b32_e32 v2, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v3, 2.0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v32, s0, v0
-; GISEL-NEXT: ds_read_b128 a[0:3], v32
-; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:16
-; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:32
-; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:48
-; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:64
-; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:80
-; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:96
-; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:112
+; GISEL-NEXT: v_add_u32_e32 v1, s0, v0
+; GISEL-NEXT: ds_read_b128 a[0:3], v1
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:112
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
-; GISEL-NEXT: v_add_u32_e32 v35, s1, v0
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; GISEL-NEXT: v_add_u32_e32 v0, s1, v0
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: s_nop 15
; GISEL-NEXT: s_nop 1
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v35, v[0:3]
-; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:112
-; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:8192
-; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:8208
-; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:8224
-; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:8240
-; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:8256
-; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:8272
-; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:8288
-; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:8304
+; GISEL-NEXT: ds_write_b128 v0, a[0:3]
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; GISEL-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:8208
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:8224
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:8240
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:8256
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:8272
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:8288
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:8304
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
-; GISEL-NEXT: v_mov_b32_e32 v35, s1
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; GISEL-NEXT: v_mov_b32_e32 v0, s1
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: s_nop 15
; GISEL-NEXT: s_nop 1
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:8304
-; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:24576
-; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:24592
-; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:24608
-; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:24624
-; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:24640
-; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:24656
-; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:24672
-; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:24688
+; GISEL-NEXT: ds_write_b128 v0, a[0:3] offset:8192
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:8224
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:8240
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:8256
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:8272
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:8288
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:8304
+; GISEL-NEXT: ds_read_b128 a[0:3], v1 offset:24576
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:24592
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:24608
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:24624
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:24640
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:24656
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:24672
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:24688
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: s_nop 15
; GISEL-NEXT: s_nop 2
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:16496
-; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:49152
-; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:49168
-; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:49184
-; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:49200
-; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:49216
-; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:49232
-; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:49248
-; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:49264
+; GISEL-NEXT: ds_write_b128 v0, a[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:16496
+; GISEL-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:49264
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
-; GISEL-NEXT: v_add_u32_e32 v32, 0xc000, v32
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v1
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: s_nop 15
; GISEL-NEXT: s_nop 1
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:24688
-; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:32768
-; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:32784
-; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:32800
-; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:32816
-; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:32832
-; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:32848
-; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:32864
-; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:32880
+; GISEL-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; GISEL-NEXT: ds_read_b128 a[0:3], v1 offset:32768
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:32784
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32800
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:32816
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:32832
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:32848
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:32864
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:32880
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: s_nop 15
; GISEL-NEXT: s_nop 2
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:32880
+; GISEL-NEXT: ds_write_b128 v0, a[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:32880
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GISEL-NEXT: s_endpgm
;
@@ -2164,348 +1843,153 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GISEL-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; GISEL-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
-; GISEL-NEXT: v_mov_b32_e32 v5, 0x32a5705f
+; GISEL-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
+; GISEL-NEXT: v_mov_b32_e32 v6, 0x32a5705f
; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mul_f32_e32 v3, s0, v2
-; GISEL-NEXT: v_fma_f32 v4, s0, v2, -v3
-; GISEL-NEXT: v_rndne_f32_e32 v6, v3
-; GISEL-NEXT: v_fmac_f32_e32 v4, s0, v5
-; GISEL-NEXT: v_sub_f32_e32 v3, v3, v6
-; GISEL-NEXT: v_add_f32_e32 v3, v3, v4
-; GISEL-NEXT: v_cvt_i32_f32_e32 v4, v6
-; GISEL-NEXT: v_exp_f32_e32 v3, v3
-; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: v_mul_f32_e32 v4, s0, v3
+; GISEL-NEXT: v_fma_f32 v5, s0, v3, -v4
+; GISEL-NEXT: v_rndne_f32_e32 v7, v4
+; GISEL-NEXT: v_fmac_f32_e32 v5, s0, v6
+; GISEL-NEXT: v_sub_f32_e32 v4, v4, v7
+; GISEL-NEXT: v_add_f32_e32 v4, v4, v5
+; GISEL-NEXT: v_cvt_i32_f32_e32 v5, v7
+; GISEL-NEXT: v_exp_f32_e32 v4, v4
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; GISEL-NEXT: s_load_dword s8, s[4:5], 0x54
; GISEL-NEXT: ; kill: killed $sgpr4_sgpr5
-; GISEL-NEXT: v_add_u32_e32 v0, s6, v32
-; GISEL-NEXT: v_ldexp_f32 v3, v3, v4
-; GISEL-NEXT: v_readfirstlane_b32 s4, v3
-; GISEL-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
-; GISEL-NEXT: ds_read_b128 a[64:67], v0
-; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:80
-; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:112
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v3
-; GISEL-NEXT: v_mov_b32_e32 v4, 0x42b17218
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GISEL-NEXT: v_add_u32_e32 v1, s6, v0
+; GISEL-NEXT: v_ldexp_f32 v4, v4, v5
+; GISEL-NEXT: v_readfirstlane_b32 s4, v4
+; GISEL-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0
+; GISEL-NEXT: ds_read_b128 a[0:3], v1
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v4
+; GISEL-NEXT: v_mov_b32_e32 v5, 0x42b17218
+; GISEL-NEXT: v_mov_b32_e32 v10, 1.0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v4
-; GISEL-NEXT: v_mul_f32_e32 v6, s1, v2
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v5
+; GISEL-NEXT: v_mul_f32_e32 v7, s1, v3
; GISEL-NEXT: s_cselect_b32 s4, 0, s4
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fma_f32 v7, s1, v2, -v6
-; GISEL-NEXT: v_rndne_f32_e32 v8, v6
+; GISEL-NEXT: v_fma_f32 v8, s1, v3, -v7
+; GISEL-NEXT: v_rndne_f32_e32 v9, v7
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s4
-; GISEL-NEXT: v_fmac_f32_e32 v7, s1, v5
-; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8
-; GISEL-NEXT: v_add_f32_e32 v6, v6, v7
-; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
-; GISEL-NEXT: v_mov_b32_e32 v8, s0
-; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208
+; GISEL-NEXT: v_fmac_f32_e32 v8, s1, v6
+; GISEL-NEXT: v_sub_f32_e32 v7, v7, v9
+; GISEL-NEXT: v_add_f32_e32 v7, v7, v8
+; GISEL-NEXT: v_cvt_i32_f32_e32 v8, v9
+; GISEL-NEXT: v_mov_b32_e32 v9, s0
+; GISEL-NEXT: ds_read_b128 a[32:35], v1 offset:8192
+; GISEL-NEXT: ds_read_b128 a[36:39], v1 offset:8208
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v8, a[64:95]
-; GISEL-NEXT: v_exp_f32_e32 v6, v6
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304
-; GISEL-NEXT: v_ldexp_f32 v6, v6, v7
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v3
-; GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v10, v9, a[0:31]
+; GISEL-NEXT: v_exp_f32_e32 v7, v7
+; GISEL-NEXT: ds_read_b128 a[40:43], v1 offset:8224
+; GISEL-NEXT: ds_read_b128 a[44:47], v1 offset:8240
+; GISEL-NEXT: ds_read_b128 a[48:51], v1 offset:8256
+; GISEL-NEXT: ds_read_b128 a[52:55], v1 offset:8272
+; GISEL-NEXT: ds_read_b128 a[56:59], v1 offset:8288
+; GISEL-NEXT: ds_read_b128 a[60:63], v1 offset:8304
+; GISEL-NEXT: v_ldexp_f32 v7, v7, v8
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v4
+; GISEL-NEXT: v_readfirstlane_b32 s0, v7
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v4
-; GISEL-NEXT: v_mul_f32_e32 v6, s2, v2
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v5
+; GISEL-NEXT: v_mul_f32_e32 v7, s2, v3
; GISEL-NEXT: s_cselect_b32 s0, 0, s0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fma_f32 v7, s2, v2, -v6
-; GISEL-NEXT: v_rndne_f32_e32 v8, v6
+; GISEL-NEXT: v_fma_f32 v8, s2, v3, -v7
+; GISEL-NEXT: v_rndne_f32_e32 v9, v7
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_fmac_f32_e32 v7, s2, v5
-; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8
-; GISEL-NEXT: v_add_f32_e32 v6, v6, v7
-; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
-; GISEL-NEXT: v_mov_b32_e32 v8, s0
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592
+; GISEL-NEXT: v_fmac_f32_e32 v8, s2, v6
+; GISEL-NEXT: v_sub_f32_e32 v7, v7, v9
+; GISEL-NEXT: v_add_f32_e32 v7, v7, v8
+; GISEL-NEXT: v_cvt_i32_f32_e32 v8, v9
+; GISEL-NEXT: v_mov_b32_e32 v9, s0
+; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:24576
+; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:24592
; GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v8, a[0:31]
-; GISEL-NEXT: v_exp_f32_e32 v6, v6
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264
-; GISEL-NEXT: v_ldexp_f32 v0, v6, v7
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v3
-; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v10, v9, a[32:63]
+; GISEL-NEXT: v_exp_f32_e32 v7, v7
+; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:24608
+; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:24624
+; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:24640
+; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:24656
+; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:24672
+; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:24688
+; GISEL-NEXT: v_add_u32_e32 v2, 0xc000, v1
+; GISEL-NEXT: ds_read_b128 a[96:99], v1 offset:49152
+; GISEL-NEXT: ds_read_b128 a[100:103], v1 offset:49168
+; GISEL-NEXT: ds_read_b128 a[104:107], v1 offset:49184
+; GISEL-NEXT: ds_read_b128 a[108:111], v1 offset:49200
+; GISEL-NEXT: ds_read_b128 a[112:115], v1 offset:49216
+; GISEL-NEXT: ds_read_b128 a[116:119], v1 offset:49232
+; GISEL-NEXT: ds_read_b128 a[120:123], v1 offset:49248
+; GISEL-NEXT: ds_read_b128 a[124:127], v1 offset:49264
+; GISEL-NEXT: v_ldexp_f32 v1, v7, v8
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v4
+; GISEL-NEXT: v_readfirstlane_b32 s0, v1
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v4
-; GISEL-NEXT: v_mul_f32_e32 v0, s3, v2
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v5
+; GISEL-NEXT: v_mul_f32_e32 v1, s3, v3
; GISEL-NEXT: s_cselect_b32 s0, 0, s0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fma_f32 v6, s3, v2, -v0
-; GISEL-NEXT: v_rndne_f32_e32 v7, v0
+; GISEL-NEXT: v_fma_f32 v7, s3, v3, -v1
+; GISEL-NEXT: v_rndne_f32_e32 v8, v1
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_fmac_f32_e32 v6, s3, v5
-; GISEL-NEXT: v_sub_f32_e32 v0, v0, v7
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v6
-; GISEL-NEXT: v_cvt_i32_f32_e32 v6, v7
-; GISEL-NEXT: v_mov_b32_e32 v7, s0
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v3
+; GISEL-NEXT: v_fmac_f32_e32 v7, s3, v6
+; GISEL-NEXT: v_sub_f32_e32 v1, v1, v8
+; GISEL-NEXT: v_add_f32_e32 v1, v1, v7
+; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
+; GISEL-NEXT: v_mov_b32_e32 v8, s0
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v4
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(8)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v7, a[32:63]
-; GISEL-NEXT: v_exp_f32_e32 v0, v0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v4
-; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
-; GISEL-NEXT: v_ldexp_f32 v0, v0, v6
-; GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GISEL-NEXT: v_mul_f32_e32 v0, s8, v2
-; GISEL-NEXT: v_fma_f32 v2, s8, v2, -v0
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v10, v8, a[64:95]
+; GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v5
+; GISEL-NEXT: ds_read_b128 a[128:131], v2 offset:32768
+; GISEL-NEXT: ds_read_b128 a[132:135], v2 offset:32784
+; GISEL-NEXT: v_ldexp_f32 v1, v1, v7
+; GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GISEL-NEXT: v_mul_f32_e32 v1, s8, v3
+; GISEL-NEXT: v_fma_f32 v3, s8, v3, -v1
; GISEL-NEXT: s_cselect_b32 s0, 0, s0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fmac_f32_e32 v2, s8, v5
-; GISEL-NEXT: v_rndne_f32_e32 v5, v0
+; GISEL-NEXT: v_fmac_f32_e32 v3, s8, v6
+; GISEL-NEXT: v_rndne_f32_e32 v6, v1
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_sub_f32_e32 v0, v0, v5
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v2
-; GISEL-NEXT: v_cvt_i32_f32_e32 v2, v5
-; GISEL-NEXT: v_mov_b32_e32 v5, s0
-; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
-; GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v5, a[96:127]
-; GISEL-NEXT: v_exp_f32_e32 v0, v0
-; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
-; GISEL-NEXT: v_ldexp_f32 v0, v0, v2
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v3
-; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: v_sub_f32_e32 v1, v1, v6
+; GISEL-NEXT: v_mov_b32_e32 v7, s0
+; GISEL-NEXT: v_add_f32_e32 v1, v1, v3
+; GISEL-NEXT: v_cvt_i32_f32_e32 v3, v6
+; GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v10, v7, a[96:127]
+; GISEL-NEXT: v_exp_f32_e32 v1, v1
+; GISEL-NEXT: ds_read_b128 a[136:139], v2 offset:32800
+; GISEL-NEXT: ds_read_b128 a[140:143], v2 offset:32816
+; GISEL-NEXT: ds_read_b128 a[144:147], v2 offset:32832
+; GISEL-NEXT: ds_read_b128 a[148:151], v2 offset:32848
+; GISEL-NEXT: ds_read_b128 a[152:155], v2 offset:32864
+; GISEL-NEXT: ds_read_b128 a[156:159], v2 offset:32880
+; GISEL-NEXT: v_ldexp_f32 v1, v1, v3
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v4
+; GISEL-NEXT: v_readfirstlane_b32 s0, v1
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v4
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v5
; GISEL-NEXT: s_cselect_b32 s0, 0, s0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_mov_b32_e32 v34, s0
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: v_add_u32_e32 v32, s7, v32
-; GISEL-NEXT: ds_write_b128 v32, v[0:3]
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: v_mov_b32_e32 v32, s7
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
+; GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GISEL-NEXT: v_add_u32_e32 v0, s7, v0
; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
@@ -2515,15 +1999,50 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ds_write_b128 v0, a[0:3]
+; GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v10, v1, a[128:159]
+; GISEL-NEXT: v_mov_b32_e32 v1, s7
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; GISEL-NEXT: ds_write_b128 v1, a[32:35] offset:8192
+; GISEL-NEXT: ds_write_b128 v1, a[36:39] offset:8208
+; GISEL-NEXT: ds_write_b128 v1, a[40:43] offset:8224
+; GISEL-NEXT: ds_write_b128 v1, a[44:47] offset:8240
+; GISEL-NEXT: ds_write_b128 v1, a[48:51] offset:8256
+; GISEL-NEXT: ds_write_b128 v1, a[52:55] offset:8272
+; GISEL-NEXT: ds_write_b128 v1, a[56:59] offset:8288
+; GISEL-NEXT: ds_write_b128 v1, a[60:63] offset:8304
+; GISEL-NEXT: ds_write_b128 v1, a[64:67] offset:16384
+; GISEL-NEXT: ds_write_b128 v1, a[68:71] offset:16400
+; GISEL-NEXT: ds_write_b128 v1, a[72:75] offset:16416
+; GISEL-NEXT: ds_write_b128 v1, a[76:79] offset:16432
+; GISEL-NEXT: ds_write_b128 v1, a[80:83] offset:16448
+; GISEL-NEXT: ds_write_b128 v1, a[84:87] offset:16464
+; GISEL-NEXT: ds_write_b128 v1, a[88:91] offset:16480
+; GISEL-NEXT: ds_write_b128 v1, a[92:95] offset:16496
+; GISEL-NEXT: ds_write_b128 v1, a[96:99] offset:24576
+; GISEL-NEXT: ds_write_b128 v1, a[100:103] offset:24592
+; GISEL-NEXT: ds_write_b128 v1, a[104:107] offset:24608
+; GISEL-NEXT: ds_write_b128 v1, a[108:111] offset:24624
+; GISEL-NEXT: ds_write_b128 v1, a[112:115] offset:24640
+; GISEL-NEXT: ds_write_b128 v1, a[116:119] offset:24656
+; GISEL-NEXT: ds_write_b128 v1, a[120:123] offset:24672
+; GISEL-NEXT: ds_write_b128 v1, a[124:127] offset:24688
+; GISEL-NEXT: ds_write_b128 v1, a[128:131] offset:32768
+; GISEL-NEXT: ds_write_b128 v1, a[132:135] offset:32784
+; GISEL-NEXT: ds_write_b128 v1, a[136:139] offset:32800
+; GISEL-NEXT: ds_write_b128 v1, a[140:143] offset:32816
+; GISEL-NEXT: ds_write_b128 v1, a[144:147] offset:32832
+; GISEL-NEXT: ds_write_b128 v1, a[148:151] offset:32848
+; GISEL-NEXT: ds_write_b128 v1, a[152:155] offset:32864
+; GISEL-NEXT: ds_write_b128 v1, a[156:159] offset:32880
; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll
index 1104362f3e995..d8a7a78d8f037 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.f64.ll
@@ -2513,19 +2513,19 @@ define amdgpu_ps <2 x i32> @s_exp_f64(double inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[6:7], s[2:3], s[6:7]
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
; SI-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], -s[4:5]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[6:7]
; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v2
; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fe62e42
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[0:1]
+; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[2:3], v[0:1], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3b39803f
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3c7abc9e
; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[2:3], v[2:3], v[0:1]
@@ -2951,22 +2951,22 @@ define amdgpu_ps <4 x i32> @s_exp_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[10:11], s[4:5], s[8:9]
; SI-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], s[10:11]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[10:11]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[12:13], |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[12:13], |v[2:3]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v5
; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[12:13]
-; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3fe62e42
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, s4
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, s5
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[4:5], v[2:3], s[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x3b39803f
-; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3c7abc9e
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[4:5], v[8:9], v[4:5]
+; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[4:5], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[4:5], v[6:7], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x6a5dcb37
; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3e5ade15
; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0xfca7ab0c
@@ -2998,11 +2998,11 @@ define amdgpu_ps <4 x i32> @s_exp_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[4:5], v[14:15], v[30:31]
; SI-GISEL-NEXT: v_mul_f64 v[0:1], s[2:3], v[0:1]
; SI-GISEL-NEXT: v_fma_f64 v[14:15], v[4:5], v[14:15], 1.0
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |v[0:1]|, v[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[14:15], 1.0
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v14, s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xc090cc00
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0xc090cc00
; SI-GISEL-NEXT: v_ldexp_f64 v[4:5], v[4:5], v14
; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v5
@@ -3010,7 +3010,7 @@ define amdgpu_ps <4 x i32> @s_exp_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[0:1], v[4:5]
; SI-GISEL-NEXT: s_or_b64 s[12:13], vcc, vcc
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[6:7]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[8:9]
; SI-GISEL-NEXT: s_mov_b32 s12, 0
; SI-GISEL-NEXT: s_mov_b32 s13, 0x7ff00000
; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[4:5], s[12:13]
@@ -3027,14 +3027,14 @@ define amdgpu_ps <4 x i32> @s_exp_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v1
; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], s[6:7]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[0:1], v[2:3], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[4:5], v[2:3], v[0:1]
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[4:5], v[8:9], v[0:1]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, s[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[4:5], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[10:11], v[12:13]
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[6:7]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[16:17]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[18:19]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[20:21]
@@ -3045,7 +3045,7 @@ define amdgpu_ps <4 x i32> @s_exp_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[30:31]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], 1.0
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], 1.0
-; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v8
+; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v6
; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[4:5], s[12:13]
@@ -3570,22 +3570,22 @@ define amdgpu_ps <6 x i32> @s_exp_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[12:13], s[10:11], s[8:9]
; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[6:7]
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], s[12:13]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[12:13]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[2:3]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v5
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
-; SI-GISEL-NEXT: s_cselect_b64 s[10:11], s[10:11], s[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3fe62e42
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, s10
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, s11
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[4:5], v[2:3], s[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x3b39803f
-; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3c7abc9e
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[10:11], v[8:9], v[4:5]
+; SI-GISEL-NEXT: s_cselect_b64 s[10:11], s[10:11], s[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[10:11], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[10:11], v[6:7], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x6a5dcb37
; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3e5ade15
; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0xfca7ab0c
@@ -3641,16 +3641,16 @@ define amdgpu_ps <6 x i32> @s_exp_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[34:35], v[32:33], s[14:15]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[14:15]
; SI-GISEL-NEXT: v_add_f64 v[34:35], v[34:35], -s[14:15]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[32:33]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[32:33]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s16, v34
; SI-GISEL-NEXT: v_readfirstlane_b32 s17, v35
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[12:13], s[16:17]
-; SI-GISEL-NEXT: v_mov_b32_e32 v33, s13
-; SI-GISEL-NEXT: v_mov_b32_e32 v32, s12
-; SI-GISEL-NEXT: v_fma_f64 v[32:33], -v[32:33], v[2:3], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, s2
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[12:13], v[2:3], v[32:33]
; SI-GISEL-NEXT: s_or_b64 s[14:15], vcc, vcc
-; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[12:13], v[8:9], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[12:13], v[6:7], v[32:33]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[34:35], v[32:33], v[10:11], v[12:13]
; SI-GISEL-NEXT: v_mul_f64 v[0:1], s[4:5], v[0:1]
@@ -3678,17 +3678,17 @@ define amdgpu_ps <6 x i32> @s_exp_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[32:33], v[0:1], s[6:7]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[14:15]
; SI-GISEL-NEXT: v_add_f64 v[32:33], v[32:33], -s[6:7]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v32
; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v33
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[12:13], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[0:1], v[2:3], s[4:5]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[6:7]
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[6:7], v[8:9], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[6:7], v[2:3], v[0:1]
; SI-GISEL-NEXT: s_or_b64 s[8:9], vcc, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[6:7], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[6:7]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[10:11], v[12:13]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[16:17]
@@ -4362,22 +4362,22 @@ define amdgpu_ps <8 x i32> @s_exp_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[14:15], s[12:13], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[8:9]
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], s[14:15]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[14:15]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[2:3]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s14, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s15, v5
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
-; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[12:13], s[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3fe62e42
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, s12
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, s13
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[4:5], v[2:3], s[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0x3b39803f
-; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3c7abc9e
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[12:13], v[8:9], v[4:5]
+; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[12:13], s[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[12:13], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x3c7abc9e
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[12:13], v[6:7], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x6a5dcb37
; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0x3e5ade15
; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0xfca7ab0c
@@ -4433,16 +4433,16 @@ define amdgpu_ps <8 x i32> @s_exp_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[34:35], v[32:33], s[16:17]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[14:15]
; SI-GISEL-NEXT: v_add_f64 v[34:35], v[34:35], -s[16:17]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[32:33]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[32:33]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s18, v34
; SI-GISEL-NEXT: v_readfirstlane_b32 s19, v35
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[14:15], s[18:19]
-; SI-GISEL-NEXT: v_mov_b32_e32 v33, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v32, s14
-; SI-GISEL-NEXT: v_fma_f64 v[32:33], -v[32:33], v[2:3], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, s2
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[14:15], v[2:3], v[32:33]
; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
-; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[14:15], v[8:9], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[14:15], v[6:7], v[32:33]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[34:35], v[32:33], v[10:11], v[12:13]
; SI-GISEL-NEXT: v_fma_f64 v[34:35], v[32:33], v[34:35], v[16:17]
@@ -4470,16 +4470,16 @@ define amdgpu_ps <8 x i32> @s_exp_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[34:35], v[32:33], s[16:17]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[14:15]
; SI-GISEL-NEXT: v_add_f64 v[34:35], v[34:35], -s[16:17]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[32:33]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[32:33]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s18, v34
; SI-GISEL-NEXT: v_readfirstlane_b32 s19, v35
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[14:15], s[18:19]
-; SI-GISEL-NEXT: v_mov_b32_e32 v33, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v32, s14
-; SI-GISEL-NEXT: v_fma_f64 v[32:33], -v[32:33], v[2:3], s[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v33, s5
+; SI-GISEL-NEXT: v_mov_b32_e32 v32, s4
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[14:15], v[2:3], v[32:33]
; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
-; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[14:15], v[8:9], v[32:33]
+; SI-GISEL-NEXT: v_fma_f64 v[32:33], -s[14:15], v[6:7], v[32:33]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[34:35], v[32:33], v[10:11], v[12:13]
; SI-GISEL-NEXT: v_mul_f64 v[0:1], s[6:7], v[0:1]
@@ -4507,17 +4507,17 @@ define amdgpu_ps <8 x i32> @s_exp_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[32:33], v[0:1], s[8:9]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[6:7], v[14:15]
; SI-GISEL-NEXT: v_add_f64 v[32:33], v[32:33], -s[8:9]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[8:9], |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[8:9], |v[0:1]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v32
; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v33
; SI-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[8:9]
; SI-GISEL-NEXT: s_cselect_b64 s[8:9], s[14:15], s[10:11]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s9
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[0:1], v[2:3], s[6:7]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[8:9]
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[8:9], v[8:9], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[8:9], v[2:3], v[0:1]
; SI-GISEL-NEXT: s_or_b64 s[10:11], vcc, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[8:9], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[10:11], v[12:13]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[16:17]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index 398f042ae5445..107dd0f78d821 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -404,8 +404,8 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x39a3b295
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_and_b32 s4, s2, 0xfffff000
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s4, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x39a3b295, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3fb8a000, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v3, s4, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll
index 22d3ed88015a9..b9f00f1774606 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.f64.ll
@@ -2735,19 +2735,19 @@ define amdgpu_ps <2 x i32> @s_exp10_f64(double inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[6:7], s[2:3], s[6:7]
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
; SI-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], -s[4:5]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[6:7]
; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v2
; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0x509f79ff
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3fd34413
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[0:1]
+; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[2:3], v[0:1], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0xa994fd21
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0xbc49dc1d
; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[2:3], v[2:3], v[0:1]
@@ -3211,22 +3211,22 @@ define amdgpu_ps <4 x i32> @s_exp10_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[10:11], s[4:5], s[8:9]
; SI-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], s[10:11]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[10:11]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[12:13], |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[12:13], |v[2:3]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v5
; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[12:13]
-; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x509f79ff
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3fd34413
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, s4
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, s5
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[4:5], v[2:3], s[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xa994fd21
-; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0xbc49dc1d
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[4:5], v[8:9], v[4:5]
+; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[4:5], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[4:5], v[6:7], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x494ea3e9
; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0xbcaf48ad
; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[4:5], v[10:11]
@@ -3264,11 +3264,11 @@ define amdgpu_ps <4 x i32> @s_exp10_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[4:5], v[18:19], v[34:35]
; SI-GISEL-NEXT: v_mul_f64 v[0:1], s[2:3], v[0:1]
; SI-GISEL-NEXT: v_fma_f64 v[18:19], v[4:5], v[18:19], 1.0
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |v[0:1]|, v[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[18:19], 1.0
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v18, s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xc090cc00
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0xc090cc00
; SI-GISEL-NEXT: v_ldexp_f64 v[4:5], v[4:5], v18
; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v5
@@ -3276,7 +3276,7 @@ define amdgpu_ps <4 x i32> @s_exp10_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x40900000
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[0:1], v[4:5]
; SI-GISEL-NEXT: s_or_b64 s[12:13], vcc, vcc
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[6:7]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[8:9]
; SI-GISEL-NEXT: s_mov_b32 s12, 0
; SI-GISEL-NEXT: s_mov_b32 s13, 0x7ff00000
; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[4:5], s[12:13]
@@ -3293,14 +3293,14 @@ define amdgpu_ps <4 x i32> @s_exp10_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v1
; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], s[6:7]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[0:1], v[2:3], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[4:5], v[2:3], v[0:1]
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[4:5], v[8:9], v[0:1]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, s[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[4:5], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[4:5]
; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[10:11]
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[6:7]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[14:15], v[2:3]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[12:13], v[16:17]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[20:21]
@@ -3313,7 +3313,7 @@ define amdgpu_ps <4 x i32> @s_exp10_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], v[34:35]
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[0:1], v[2:3], 1.0
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], 1.0
-; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v8
+; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v6
; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[4:5], s[12:13]
@@ -3880,22 +3880,22 @@ define amdgpu_ps <6 x i32> @s_exp10_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[12:13], s[10:11], s[8:9]
; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[6:7]
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], s[12:13]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[12:13]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[2:3]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v5
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
-; SI-GISEL-NEXT: s_cselect_b64 s[10:11], s[10:11], s[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x509f79ff
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3fd34413
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, s10
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, s11
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[4:5], v[2:3], s[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xa994fd21
-; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0xbc49dc1d
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[10:11], v[8:9], v[4:5]
+; SI-GISEL-NEXT: s_cselect_b64 s[10:11], s[10:11], s[12:13]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[10:11], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[10:11], v[6:7], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x494ea3e9
; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0xbcaf48ad
; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[4:5], v[10:11]
@@ -3957,16 +3957,16 @@ define amdgpu_ps <6 x i32> @s_exp10_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[38:39], v[36:37], s[14:15]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[18:19]
; SI-GISEL-NEXT: v_add_f64 v[38:39], v[38:39], -s[14:15]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[36:37]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |v[36:37]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s16, v38
; SI-GISEL-NEXT: v_readfirstlane_b32 s17, v39
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[12:13], s[16:17]
-; SI-GISEL-NEXT: v_mov_b32_e32 v37, s13
-; SI-GISEL-NEXT: v_mov_b32_e32 v36, s12
-; SI-GISEL-NEXT: v_fma_f64 v[36:37], -v[36:37], v[2:3], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, s3
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, s2
+; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[12:13], v[2:3], v[36:37]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v40, s[12:13]
-; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[12:13], v[8:9], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[12:13], v[6:7], v[36:37]
; SI-GISEL-NEXT: s_or_b64 s[14:15], vcc, vcc
; SI-GISEL-NEXT: v_mul_f64 v[38:39], v[36:37], v[10:11]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[4:5]
@@ -3996,17 +3996,17 @@ define amdgpu_ps <6 x i32> @s_exp10_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[36:37], v[0:1], s[6:7]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[18:19]
; SI-GISEL-NEXT: v_add_f64 v[36:37], v[36:37], -s[6:7]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v36
; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v37
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[12:13], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[0:1], v[2:3], s[4:5]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[6:7]
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[6:7], v[8:9], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[6:7], v[2:3], v[0:1]
; SI-GISEL-NEXT: s_or_b64 s[8:9], vcc, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[6:7], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[6:7]
; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[10:11]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[14:15], v[2:3]
@@ -4734,22 +4734,22 @@ define amdgpu_ps <8 x i32> @s_exp10_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_and_b64 s[14:15], s[12:13], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[8:9]
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], s[14:15]
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0x432fffff
+; SI-GISEL-NEXT: v_mov_b32_e32 v8, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x432fffff
; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[14:15]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[2:3]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[2:3]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s14, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s15, v5
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
-; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[12:13], s[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, s1
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x509f79ff
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3fd34413
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, s12
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, s13
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -v[4:5], v[2:3], s[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xa994fd21
-; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0xbc49dc1d
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[12:13], v[8:9], v[4:5]
+; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[12:13], s[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[12:13], v[2:3], v[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, 0xa994fd21
+; SI-GISEL-NEXT: v_mov_b32_e32 v7, 0xbc49dc1d
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], -s[12:13], v[6:7], v[4:5]
; SI-GISEL-NEXT: v_mov_b32_e32 v10, 0x494ea3e9
; SI-GISEL-NEXT: v_mov_b32_e32 v11, 0xbcaf48ad
; SI-GISEL-NEXT: v_mul_f64 v[12:13], v[4:5], v[10:11]
@@ -4811,16 +4811,16 @@ define amdgpu_ps <8 x i32> @s_exp10_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[38:39], v[36:37], s[16:17]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[18:19]
; SI-GISEL-NEXT: v_add_f64 v[38:39], v[38:39], -s[16:17]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[36:37]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[36:37]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s18, v38
; SI-GISEL-NEXT: v_readfirstlane_b32 s19, v39
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[14:15], s[18:19]
-; SI-GISEL-NEXT: v_mov_b32_e32 v37, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v36, s14
-; SI-GISEL-NEXT: v_fma_f64 v[36:37], -v[36:37], v[2:3], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, s3
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, s2
+; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[14:15], v[2:3], v[36:37]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v40, s[14:15]
-; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[14:15], v[8:9], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[14:15], v[6:7], v[36:37]
; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
; SI-GISEL-NEXT: v_mul_f64 v[38:39], v[36:37], v[10:11]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[4:5]
@@ -4850,16 +4850,16 @@ define amdgpu_ps <8 x i32> @s_exp10_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[38:39], v[36:37], s[16:17]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[18:19]
; SI-GISEL-NEXT: v_add_f64 v[38:39], v[38:39], -s[16:17]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[36:37]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |v[36:37]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s18, v38
; SI-GISEL-NEXT: v_readfirstlane_b32 s19, v39
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[14:15], s[18:19]
-; SI-GISEL-NEXT: v_mov_b32_e32 v37, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v36, s14
-; SI-GISEL-NEXT: v_fma_f64 v[36:37], -v[36:37], v[2:3], s[4:5]
+; SI-GISEL-NEXT: v_mov_b32_e32 v37, s5
+; SI-GISEL-NEXT: v_mov_b32_e32 v36, s4
+; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[14:15], v[2:3], v[36:37]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v40, s[14:15]
-; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[14:15], v[8:9], v[36:37]
+; SI-GISEL-NEXT: v_fma_f64 v[36:37], -s[14:15], v[6:7], v[36:37]
; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
; SI-GISEL-NEXT: v_mul_f64 v[38:39], v[36:37], v[10:11]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[4:5]
@@ -4889,17 +4889,17 @@ define amdgpu_ps <8 x i32> @s_exp10_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_add_f64 v[36:37], v[0:1], s[8:9]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[6:7], v[18:19]
; SI-GISEL-NEXT: v_add_f64 v[36:37], v[36:37], -s[8:9]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[8:9], |v[0:1]|, v[6:7]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[8:9], |v[0:1]|, v[8:9]
; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v36
; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v37
; SI-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[8:9]
; SI-GISEL-NEXT: s_cselect_b64 s[8:9], s[14:15], s[10:11]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s9
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -v[0:1], v[2:3], s[6:7]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[8:9]
-; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[8:9], v[8:9], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[8:9], v[2:3], v[0:1]
; SI-GISEL-NEXT: s_or_b64 s[10:11], vcc, vcc
+; SI-GISEL-NEXT: v_fma_f64 v[0:1], -s[8:9], v[6:7], v[0:1]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[8:9]
; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[10:11]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[14:15], v[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index 6dd6ae3eca990..176b8ae0c69f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -406,8 +406,8 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3a2784bc
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_and_b32 s4, s2, 0xfffff000
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s4, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3a2784bc, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x40549000, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v3, s4, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll
index 69bf598973247..5c61e1809c6ac 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.f64.ll
@@ -2458,20 +2458,18 @@ define amdgpu_ps <2 x i32> @s_exp2_f64(double inreg %in) #0 {
; SI-GISEL-NEXT: s_mov_b32 s3, 0x43300000
; SI-GISEL-NEXT: s_and_b64 s[4:5], s[0:1], s[4:5]
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x432fffff
-; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[2:3]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[4:5], |s[0:1]|, v[2:3]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], s[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], -s[2:3]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[4:5], |s[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v3
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[0:1], s[2:3]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], -v[0:1]
+; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[2:3]
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3b39803f
; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3c7abc9e
; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
@@ -2880,23 +2878,21 @@ define amdgpu_ps <4 x i32> @s_exp2_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_mov_b32 s5, 0x43300000
; SI-GISEL-NEXT: s_and_b64 s[8:9], s[0:1], s[6:7]
; SI-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s9
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x432fffff
-; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[8:9]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |s[0:1]|, v[2:3]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], s[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], -s[8:9]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |s[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v3
; SI-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[10:11]
; SI-GISEL-NEXT: s_cselect_b64 s[8:9], s[0:1], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s9
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], -v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x3b39803f
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x3c7abc9e
-; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[8:9]
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[2:3]
; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[6:7]
@@ -2929,20 +2925,20 @@ define amdgpu_ps <4 x i32> @s_exp2_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_mov_b32_e32 v28, 11
; SI-GISEL-NEXT: v_mov_b32_e32 v29, 0x3fe00000
; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[0:1], v[12:13], v[28:29]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |s[2:3]|, v[2:3]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v30, s[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[0:1], v[12:13], 1.0
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[10:11], |s[2:3]|, v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[12:13], 1.0
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v12, s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0xc090cc00
-; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v12
+; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0
+; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v30
+; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x40900000
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[0:1], v[12:13]
; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v1
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x40900000
-; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[0:1], v[0:1]
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0xc090cc00
; SI-GISEL-NEXT: s_or_b64 s[12:13], vcc, vcc
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[2:3]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[0:1]
; SI-GISEL-NEXT: s_mov_b32 s12, 0
; SI-GISEL-NEXT: s_mov_b32 s13, 0x7ff00000
; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[8:9], s[12:13]
@@ -2950,37 +2946,35 @@ define amdgpu_ps <4 x i32> @s_exp2_v2f64(<2 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; SI-GISEL-NEXT: s_and_b64 s[6:7], s[2:3], s[6:7]
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v13, s5
-; SI-GISEL-NEXT: v_mov_b32_e32 v12, s4
-; SI-GISEL-NEXT: v_add_f64 v[12:13], s[2:3], v[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s2
+; SI-GISEL-NEXT: v_add_f64 v[30:31], v[4:5], s[4:5]
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[10:11], s[10:11]
-; SI-GISEL-NEXT: v_add_f64 v[12:13], v[12:13], -s[4:5]
-; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[0:1]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v12
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v13
+; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[4:5]
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[12:13]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v30
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v31
; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[2:3], s[4:5]
-; SI-GISEL-NEXT: v_mov_b32_e32 v13, s5
-; SI-GISEL-NEXT: v_mov_b32_e32 v12, s4
-; SI-GISEL-NEXT: v_add_f64 v[12:13], s[2:3], -v[12:13]
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[4:5]
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
-; SI-GISEL-NEXT: v_mul_f64 v[4:5], v[12:13], v[4:5]
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[2:3]
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v8, s[4:5]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[10:11]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[14:15]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[16:17]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[18:19]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[20:21]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[22:23]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[24:25]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[26:27]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], v[28:29]
-; SI-GISEL-NEXT: v_fma_f64 v[6:7], v[4:5], v[6:7], 1.0
-; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], 1.0
-; SI-GISEL-NEXT: v_ldexp_f64 v[4:5], v[4:5], v8
-; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v4
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v5
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[4:5], v[2:3]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[0:1]
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[8:9], v[2:3]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[14:15]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[16:17]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[18:19]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[20:21]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[22:23]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[24:25]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[26:27]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[28:29]
+; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], 1.0
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[2:3], v[4:5], 1.0
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v6
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[4:5], s[12:13]
; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], 0
@@ -3475,23 +3469,21 @@ define amdgpu_ps <6 x i32> @s_exp2_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_mov_b32 s7, 0x43300000
; SI-GISEL-NEXT: s_and_b64 s[10:11], s[0:1], s[8:9]
; SI-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s10
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s11
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x432fffff
-; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[10:11]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[12:13], |s[0:1]|, v[2:3]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], s[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], -s[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[12:13], |s[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v3
; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[12:13]
; SI-GISEL-NEXT: s_cselect_b64 s[10:11], s[0:1], s[10:11]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s10
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s11
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], -v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x3b39803f
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x3c7abc9e
-; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[2:3]
; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[6:7]
@@ -3526,44 +3518,45 @@ define amdgpu_ps <6 x i32> @s_exp2_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[0:1], v[12:13], v[28:29]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v30, s[10:11]
; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[0:1], v[12:13], 1.0
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |s[2:3]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, s3
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[12:13], 1.0
; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0
; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v30
; SI-GISEL-NEXT: v_mov_b32_e32 v13, 0x40900000
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[0:1], v[12:13]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v1
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0xc090cc00
-; SI-GISEL-NEXT: s_or_b64 s[10:11], vcc, vcc
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[0:1]
-; SI-GISEL-NEXT: s_mov_b32 s10, 0
-; SI-GISEL-NEXT: s_mov_b32 s11, 0x7ff00000
-; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[12:13], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[12:13], vcc, vcc
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[0:1], v[0:1]
+; SI-GISEL-NEXT: s_mov_b32 s12, 0
+; SI-GISEL-NEXT: s_mov_b32 s13, 0x7ff00000
+; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[10:11], s[12:13]
+; SI-GISEL-NEXT: s_or_b64 s[10:11], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
-; SI-GISEL-NEXT: s_and_b64 s[12:13], s[2:3], s[8:9]
-; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[6:7]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s13
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s12
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[2:3], v[30:31]
+; SI-GISEL-NEXT: s_and_b64 s[10:11], s[2:3], s[8:9]
+; SI-GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[6:7]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, s2
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[30:31], s[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |s[2:3]|, v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[32:33], -s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
-; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[12:13]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v32
+; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v33
+; SI-GISEL-NEXT: s_cselect_b64 s[10:11], s[2:3], s[10:11]
+; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[10:11]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[12:13]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v30
-; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v31
-; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[2:3], s[12:13]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s13
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s12
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[2:3], -v[30:31]
-; SI-GISEL-NEXT: s_or_b64 s[14:15], vcc, vcc
-; SI-GISEL-NEXT: v_mul_f64 v[32:33], v[30:31], v[4:5]
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[0:1]
+; SI-GISEL-NEXT: v_mul_f64 v[32:33], v[30:31], v[2:3]
+; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[30:31], v[8:9], v[32:33]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[0:1]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |s[4:5]|, v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s4
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[16:17]
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, s5
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[18:19]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[20:21]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[22:23]
@@ -3572,32 +3565,27 @@ define amdgpu_ps <6 x i32> @s_exp2_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[28:29]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], 1.0
; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[30:31], v[32:33], 1.0
-; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v32, s[12:13]
+; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v32, s[10:11]
; SI-GISEL-NEXT: v_ldexp_f64 v[30:31], v[30:31], v32
-; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v30
-; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v31
-; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[12:13], s[10:11]
-; SI-GISEL-NEXT: s_or_b64 s[12:13], vcc, vcc
+; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v30
+; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v31
+; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[10:11], s[12:13]
+; SI-GISEL-NEXT: s_or_b64 s[10:11], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], 0
; SI-GISEL-NEXT: s_and_b64 s[8:9], s[4:5], s[8:9]
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[8:9], s[6:7]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s7
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s6
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[4:5], v[30:31]
-; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[12:13]
+; SI-GISEL-NEXT: v_add_f64 v[30:31], v[4:5], s[6:7]
+; SI-GISEL-NEXT: s_or_b64 s[8:9], s[14:15], s[14:15]
; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[6:7]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[6:7], |s[4:5]|, v[2:3]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v30
-; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v31
-; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[4:5], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s6
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, s7
-; SI-GISEL-NEXT: v_add_f64 v[2:3], s[4:5], -v[2:3]
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[12:13]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v30
+; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v31
+; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[4:5], s[6:7]
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[6:7]
; SI-GISEL-NEXT: s_or_b64 s[8:9], vcc, vcc
-; SI-GISEL-NEXT: v_mul_f64 v[4:5], v[2:3], v[4:5]
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[4:5], v[2:3]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
-; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[2:3], v[8:9], v[4:5]
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[8:9], v[2:3]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[6:7], v[10:11]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[6:7]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[14:15]
@@ -3613,7 +3601,7 @@ define amdgpu_ps <6 x i32> @s_exp2_v3f64(<3 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v6
; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v3
-; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[6:7], s[10:11]
+; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[6:7], s[12:13]
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], 0
; SI-GISEL-NEXT: ; return to shader part epilog
@@ -4239,23 +4227,21 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_mov_b32 s9, 0x43300000
; SI-GISEL-NEXT: s_and_b64 s[12:13], s[0:1], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[12:13], s[12:13], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s12
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s13
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, -1
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x432fffff
-; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[12:13]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |s[0:1]|, v[2:3]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[0:1], s[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, -1
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x432fffff
+; SI-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], -s[12:13]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[14:15], |s[0:1]|, v[4:5]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s12, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s13, v3
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[14:15]
; SI-GISEL-NEXT: s_cselect_b64 s[12:13], s[0:1], s[12:13]
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s12
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s13
-; SI-GISEL-NEXT: v_add_f64 v[0:1], s[0:1], -v[0:1]
-; SI-GISEL-NEXT: v_mov_b32_e32 v4, 0x3b39803f
-; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0x3c7abc9e
-; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], -s[12:13]
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3b39803f
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x3c7abc9e
+; SI-GISEL-NEXT: v_mul_f64 v[6:7], v[0:1], v[2:3]
; SI-GISEL-NEXT: v_mov_b32_e32 v8, 0xfefa39ef
; SI-GISEL-NEXT: v_mov_b32_e32 v9, 0x3fe62e42
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[6:7]
@@ -4290,7 +4276,7 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[0:1], v[12:13], v[28:29]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v30, s[12:13]
; SI-GISEL-NEXT: v_fma_f64 v[12:13], v[0:1], v[12:13], 1.0
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |s[2:3]|, v[2:3]
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, s3
; SI-GISEL-NEXT: v_fma_f64 v[0:1], v[0:1], v[12:13], 1.0
; SI-GISEL-NEXT: v_mov_b32_e32 v12, 0
; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[0:1], v30
@@ -4309,24 +4295,22 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; SI-GISEL-NEXT: s_and_b64 s[14:15], s[2:3], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s14
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[2:3], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, s2
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[30:31], s[14:15]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |s[2:3]|, v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[32:33], -s[14:15]
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s14, v32
+; SI-GISEL-NEXT: v_readfirstlane_b32 s15, v33
+; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[2:3], s[14:15]
; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[14:15]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[2:3], v[12:13]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s14, v30
-; SI-GISEL-NEXT: v_readfirstlane_b32 s15, v31
-; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[2:3], s[14:15]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s14
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[2:3], -v[30:31]
+; SI-GISEL-NEXT: v_mul_f64 v[32:33], v[30:31], v[2:3]
; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
-; SI-GISEL-NEXT: v_mul_f64 v[32:33], v[30:31], v[4:5]
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[0:1]
; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[30:31], v[8:9], v[32:33]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |s[4:5]|, v[2:3]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[2:3], v[0:1]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |s[4:5]|, v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[14:15]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[16:17]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[18:19]
@@ -4346,24 +4330,25 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_cselect_b64 s[2:3], s[2:3], 0
; SI-GISEL-NEXT: s_and_b64 s[14:15], s[4:5], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[14:15], s[14:15], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s14
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[4:5], v[30:31]
+; SI-GISEL-NEXT: v_mov_b32_e32 v31, s5
+; SI-GISEL-NEXT: v_mov_b32_e32 v30, s4
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[30:31], s[14:15]
; SI-GISEL-NEXT: s_or_b64 s[16:17], s[16:17], s[16:17]
-; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[14:15]
+; SI-GISEL-NEXT: v_add_f64 v[32:33], v[32:33], -s[14:15]
; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[12:13]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s14, v30
-; SI-GISEL-NEXT: v_readfirstlane_b32 s15, v31
+; SI-GISEL-NEXT: v_readfirstlane_b32 s14, v32
+; SI-GISEL-NEXT: v_readfirstlane_b32 s15, v33
; SI-GISEL-NEXT: s_cselect_b64 s[14:15], s[4:5], s[14:15]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s15
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s14
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[4:5], -v[30:31]
-; SI-GISEL-NEXT: s_or_b64 s[16:17], vcc, vcc
-; SI-GISEL-NEXT: v_mul_f64 v[32:33], v[30:31], v[4:5]
+; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[14:15]
+; SI-GISEL-NEXT: s_or_b64 s[18:19], vcc, vcc
+; SI-GISEL-NEXT: v_mul_f64 v[32:33], v[30:31], v[2:3]
; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[4:5], v[0:1]
; SI-GISEL-NEXT: v_fma_f64 v[30:31], v[30:31], v[8:9], v[32:33]
+; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[16:17], |s[6:7]|, v[4:5]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[6:7], v[10:11]
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s6
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[14:15]
+; SI-GISEL-NEXT: v_mov_b32_e32 v5, s7
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[16:17]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[18:19]
; SI-GISEL-NEXT: v_fma_f64 v[32:33], v[30:31], v[32:33], v[20:21]
@@ -4382,26 +4367,21 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: s_cselect_b64 s[4:5], s[4:5], 0
; SI-GISEL-NEXT: s_and_b64 s[10:11], s[6:7], s[10:11]
; SI-GISEL-NEXT: s_or_b64 s[8:9], s[10:11], s[8:9]
-; SI-GISEL-NEXT: v_mov_b32_e32 v31, s9
-; SI-GISEL-NEXT: v_mov_b32_e32 v30, s8
-; SI-GISEL-NEXT: v_add_f64 v[30:31], s[6:7], v[30:31]
-; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[6:7], v[12:13]
+; SI-GISEL-NEXT: v_add_f64 v[30:31], v[4:5], s[8:9]
+; SI-GISEL-NEXT: s_or_b64 s[10:11], s[16:17], s[16:17]
; SI-GISEL-NEXT: v_add_f64 v[30:31], v[30:31], -s[8:9]
-; SI-GISEL-NEXT: v_cmp_gt_f64_e64 s[8:9], |s[6:7]|, v[2:3]
-; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v30
-; SI-GISEL-NEXT: v_readfirstlane_b32 s11, v31
-; SI-GISEL-NEXT: s_or_b64 s[8:9], s[8:9], s[8:9]
-; SI-GISEL-NEXT: s_cselect_b64 s[8:9], s[6:7], s[10:11]
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s8
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, s9
-; SI-GISEL-NEXT: v_add_f64 v[2:3], s[6:7], -v[2:3]
-; SI-GISEL-NEXT: v_cmp_nlt_f64_e64 s[6:7], s[6:7], v[0:1]
-; SI-GISEL-NEXT: v_mul_f64 v[4:5], v[2:3], v[4:5]
-; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[2:3], v[8:9], v[4:5]
+; SI-GISEL-NEXT: v_cmp_ngt_f64_e32 vcc, s[6:7], v[12:13]
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v30
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v31
+; SI-GISEL-NEXT: s_cselect_b64 s[8:9], s[6:7], s[8:9]
+; SI-GISEL-NEXT: v_add_f64 v[4:5], v[4:5], -s[8:9]
+; SI-GISEL-NEXT: s_or_b64 s[10:11], vcc, vcc
+; SI-GISEL-NEXT: v_mul_f64 v[2:3], v[4:5], v[2:3]
+; SI-GISEL-NEXT: v_cmp_nlt_f64_e32 vcc, s[6:7], v[0:1]
+; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[4:5], v[8:9], v[2:3]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[6:7], v[10:11]
; SI-GISEL-NEXT: v_cvt_i32_f64_e32 v6, s[8:9]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[14:15]
-; SI-GISEL-NEXT: s_or_b64 s[8:9], vcc, vcc
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[16:17]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[18:19]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[20:21]
@@ -4411,12 +4391,12 @@ define amdgpu_ps <8 x i32> @s_exp2_v4f64(<4 x double> inreg %in) #0 {
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], v[28:29]
; SI-GISEL-NEXT: v_fma_f64 v[4:5], v[2:3], v[4:5], 1.0
; SI-GISEL-NEXT: v_fma_f64 v[2:3], v[2:3], v[4:5], 1.0
-; SI-GISEL-NEXT: v_ldexp_f64 v[0:1], v[2:3], v6
-; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v1
-; SI-GISEL-NEXT: s_cselect_b64 s[8:9], s[8:9], s[12:13]
-; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[8:9], 0
+; SI-GISEL-NEXT: v_ldexp_f64 v[2:3], v[2:3], v6
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v3
+; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[8:9], s[12:13]
+; SI-GISEL-NEXT: s_or_b64 s[8:9], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b64 s[6:7], s[6:7], 0
; SI-GISEL-NEXT: ; return to shader part epilog
;
; VI-SDAG-LABEL: s_exp2_v4f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 27bb8ac56706d..65d6b07bb7f3b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -31,18 +31,18 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
;
; SI-GISEL-LABEL: s_exp2_f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-GISEL-NEXT: s_load_dword s0, s[4:5], 0xb
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; SI-GISEL-NEXT: v_add_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: v_add_f32_e32 v0, s0, v1
; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s3
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -76,9 +76,9 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s3, 0x42800000, 0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_add_f32_e32 v0, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
+; VI-GISEL-NEXT: v_add_f32_e32 v0, s2, v1
; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
; VI-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -109,16 +109,16 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
; GFX900-GISEL: ; %bb.0:
; GFX900-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s1, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s0, v1
; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s2
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -200,28 +200,28 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; SI-GISEL-LABEL: s_exp2_v2f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s7, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; SI-GISEL-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
+; SI-GISEL-NEXT: v_add_f32_e32 v1, s2, v1
; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
-; SI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; SI-GISEL-NEXT: v_add_f32_e32 v0, s6, v0
-; SI-GISEL-NEXT: v_add_f32_e32 v1, s7, v1
-; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_add_f32_e32 v0, s2, v2
; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s3
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s0
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v0
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s4
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: s_exp2_v2f32:
@@ -251,27 +251,27 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; VI-GISEL-LABEL: s_exp2_v2f32:
; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s7, v0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
-; VI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; VI-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; VI-GISEL-NEXT: v_add_f32_e32 v0, s6, v0
-; VI-GISEL-NEXT: v_add_f32_e32 v1, s7, v1
-; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
+; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; VI-GISEL-NEXT: v_add_f32_e32 v1, s2, v1
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
+; VI-GISEL-NEXT: v_add_f32_e32 v0, s2, v2
; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
-; VI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s3
-; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; VI-GISEL-NEXT: v_exp_f32_e32 v2, v0
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s4
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v2, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-GISEL-NEXT: s_endpgm
;
@@ -301,27 +301,27 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; GFX900-GISEL-LABEL: s_exp2_v2f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v0
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s11, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, s11, v1
-; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
+; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, s2, v1
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s2, v2
; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
-; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s3
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s0
-; GFX900-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s4
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v2, s2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX900-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX900-GISEL-NEXT: s_endpgm
;
; R600-LABEL: s_exp2_v2f32:
@@ -424,35 +424,35 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
;
; SI-GISEL-LABEL: s_exp2_v3f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd
; SI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v0
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v0
-; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; SI-GISEL-NEXT: v_add_f32_e32 v1, s0, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; SI-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-GISEL-NEXT: v_add_f32_e32 v2, s0, v2
+; SI-GISEL-NEXT: s_cselect_b32 s6, 0xffffffc0, 0
; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, s2
; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; SI-GISEL-NEXT: v_add_f32_e32 v1, s8, v1
-; SI-GISEL-NEXT: v_add_f32_e32 v2, s9, v2
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
-; SI-GISEL-NEXT: v_add_f32_e32 v0, s10, v0
+; SI-GISEL-NEXT: v_add_f32_e32 v0, s0, v3
; SI-GISEL-NEXT: v_exp_f32_e32 v3, v0
; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s3
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s2
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s6
; SI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, -1
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
; SI-GISEL-NEXT: v_ldexp_f32_e64 v2, v3, s0
; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-GISEL-NEXT: buffer_store_dword v2, off, s[4:7], 0 offset:8
@@ -494,36 +494,36 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
;
; VI-GISEL-LABEL: s_exp2_v3f32:
; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; VI-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; VI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v0
-; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x42800000, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 0xffffffc0, 0
-; VI-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
+; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: v_add_f32_e32 v1, s0, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-GISEL-NEXT: v_add_f32_e32 v2, s0, v2
; VI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s2
; VI-GISEL-NEXT: s_cselect_b32 s1, 0x42800000, 0
-; VI-GISEL-NEXT: v_add_f32_e32 v1, s8, v1
-; VI-GISEL-NEXT: v_add_f32_e32 v2, s9, v2
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
; VI-GISEL-NEXT: v_exp_f32_e32 v2, v2
-; VI-GISEL-NEXT: v_add_f32_e32 v0, s10, v0
+; VI-GISEL-NEXT: v_add_f32_e32 v0, s1, v3
; VI-GISEL-NEXT: v_exp_f32_e32 v3, v0
-; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; VI-GISEL-NEXT: v_ldexp_f32 v1, v2, s0
; VI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; VI-GISEL-NEXT: v_ldexp_f32 v2, v3, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v4, s3
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s4
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, s5
; VI-GISEL-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
; VI-GISEL-NEXT: s_endpgm
;
@@ -561,36 +561,36 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
;
; GFX900-GISEL-LABEL: s_exp2_v3f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v0
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x42800000, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 0xffffffc0, 0
-; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, s0, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v2, s0, v2
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, s2
; GFX900-GISEL-NEXT: s_cselect_b32 s1, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, s8, v1
-; GFX900-GISEL-NEXT: v_add_f32_e32 v2, s9, v2
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s1
; GFX900-GISEL-NEXT: v_exp_f32_e32 v1, v1
; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v2
-; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s1, v3
; GFX900-GISEL-NEXT: v_exp_f32_e32 v3, v0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v2, s0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; GFX900-GISEL-NEXT: v_ldexp_f32 v2, v3, s0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX900-GISEL-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3]
+; GFX900-GISEL-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7]
; GFX900-GISEL-NEXT: s_endpgm
;
; R600-LABEL: s_exp2_v3f32:
@@ -723,44 +723,44 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; SI-GISEL-LABEL: s_exp2_v4f32:
; SI-GISEL: ; %bb.0:
; SI-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; SI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v1
; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v1
-; SI-GISEL-NEXT: v_add_f32_e32 v0, s8, v0
-; SI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
+; SI-GISEL-NEXT: v_add_f32_e32 v0, s2, v0
; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s9, v1
+; SI-GISEL-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s9
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; SI-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s4
+; SI-GISEL-NEXT: v_add_f32_e32 v2, s2, v2
+; SI-GISEL-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, s10
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s11, v1
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s3
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, s0
-; SI-GISEL-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; SI-GISEL-NEXT: v_add_f32_e32 v2, s9, v2
-; SI-GISEL-NEXT: v_add_f32_e32 v3, s10, v3
-; SI-GISEL-NEXT: v_add_f32_e32 v1, s11, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v3, s2, v3
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0xffffffc0, 0
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s11
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
+; SI-GISEL-NEXT: v_add_f32_e32 v1, s2, v4
; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
; SI-GISEL-NEXT: v_exp_f32_e32 v3, v3
; SI-GISEL-NEXT: v_exp_f32_e32 v4, v1
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s2
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v2, v3, s3
-; SI-GISEL-NEXT: v_ldexp_f32_e64 v3, v4, s0
-; SI-GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s4
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v2, v3, s5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v3, v4, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: s_exp2_v4f32:
@@ -805,44 +805,44 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
;
; VI-GISEL-LABEL: s_exp2_v4f32:
; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; VI-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; VI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v1
-; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v1
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x42800000, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 0xffffffc0, 0
-; VI-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: v_add_f32_e32 v0, s0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
; VI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; VI-GISEL-NEXT: v_add_f32_e32 v2, s0, v2
+; VI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s2
; VI-GISEL-NEXT: s_cselect_b32 s1, 0x42800000, 0
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s11, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
+; VI-GISEL-NEXT: v_add_f32_e32 v3, s1, v3
; VI-GISEL-NEXT: s_cselect_b32 s1, 0xffffffc0, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x42800000, 0
-; VI-GISEL-NEXT: v_add_f32_e32 v2, s9, v2
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; VI-GISEL-NEXT: v_add_f32_e32 v0, s8, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, s3
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
; VI-GISEL-NEXT: v_exp_f32_e32 v2, v2
-; VI-GISEL-NEXT: v_add_f32_e32 v3, s10, v3
-; VI-GISEL-NEXT: v_add_f32_e32 v1, s11, v1
-; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_add_f32_e32 v1, s2, v4
; VI-GISEL-NEXT: v_exp_f32_e32 v3, v3
; VI-GISEL-NEXT: v_exp_f32_e32 v4, v1
; VI-GISEL-NEXT: v_ldexp_f32 v1, v2, s0
; VI-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
-; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s5
; VI-GISEL-NEXT: v_ldexp_f32 v2, v3, s1
; VI-GISEL-NEXT: v_ldexp_f32 v3, v4, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; VI-GISEL-NEXT: v_mov_b32_e32 v4, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; VI-GISEL-NEXT: v_mov_b32_e32 v5, s5
; VI-GISEL-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-GISEL-NEXT: s_endpgm
;
@@ -887,44 +887,44 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
;
; GFX900-GISEL-LABEL: s_exp2_v4f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x42800000, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 0xffffffc0, 0
-; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v2, s0, v2
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, s2
; GFX900-GISEL-NEXT: s_cselect_b32 s1, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s11, v1
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v3, s1, v3
; GFX900-GISEL-NEXT: s_cselect_b32 s1, 0xffffffc0, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x42800000, 0
-; GFX900-GISEL-NEXT: v_add_f32_e32 v2, s9, v2
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-GISEL-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x42800000, 0
; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v2
-; GFX900-GISEL-NEXT: v_add_f32_e32 v3, s10, v3
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, s11, v1
-; GFX900-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, s2, v4
; GFX900-GISEL-NEXT: v_exp_f32_e32 v3, v3
; GFX900-GISEL-NEXT: v_exp_f32_e32 v4, v1
; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v2, s0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s5
; GFX900-GISEL-NEXT: v_ldexp_f32 v2, v3, s1
; GFX900-GISEL-NEXT: v_ldexp_f32 v3, v4, s0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX900-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
+; GFX900-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX900-GISEL-NEXT: s_endpgm
;
; R600-LABEL: s_exp2_v4f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index ee51803a1df50..5d8d70c960ce0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -45,36 +45,37 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; SI-GISEL-LABEL: s_log_f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dword s0, s[4:5], 0xb
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s6, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s0
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
-; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317217, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
@@ -115,25 +116,25 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3805fdf4
; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3f317000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3805fdf4
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v3, s2, v0
; VI-GISEL-NEXT: v_mul_f32_e32 v1, s2, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v3
-; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v1, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
@@ -182,35 +183,36 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-GISEL-LABEL: s_log_f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317217, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX900-GISEL-NEXT: s_endpgm
;
@@ -378,36 +380,37 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; SI-GISEL-LABEL: s_log_contract_f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dword s0, s[4:5], 0xb
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s6, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s0
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
-; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317217, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
@@ -448,25 +451,25 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3805fdf4
; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3f317000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3805fdf4
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v3, s2, v0
; VI-GISEL-NEXT: v_mul_f32_e32 v1, s2, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v3
-; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v1, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
@@ -515,35 +518,36 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; GFX900-GISEL-LABEL: s_log_contract_f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3377d1cf
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317217, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f317217
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317217, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX900-GISEL-NEXT: s_endpgm
;
@@ -736,42 +740,42 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s2, v1
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s4
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v1
; SI-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; SI-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; SI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
-; SI-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v5
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s7, s2
-; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x41b17218, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s3, v0
+; SI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0x41b17218, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s2
; SI-GISEL-NEXT: v_log_f32_e32 v1, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; SI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; SI-GISEL-NEXT: v_sub_f32_e32 v0, s4, v0
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v1
; SI-GISEL-NEXT: v_fma_f32 v2, v1, v2, -v5
; SI-GISEL-NEXT: v_fma_f32 v2, v1, v3, v2
; SI-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v1|, v4
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v1, s2, v1
@@ -840,38 +844,38 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s2, v1
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s2, v1
-; VI-GISEL-NEXT: s_and_b32 s4, s2, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; VI-GISEL-NEXT: s_and_b32 s4, s6, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v4, s4, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v5, s4, v2
-; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x3805fdf4, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
-; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v7
-; VI-GISEL-NEXT: v_mul_f32_e32 v6, s4, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; VI-GISEL-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-GISEL-NEXT: v_add_f32_e32 v4, v6, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v5, s4, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v4, v5, v4
; VI-GISEL-NEXT: v_readfirstlane_b32 s7, v4
; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x7f800000
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; VI-GISEL-NEXT: s_cselect_b32 s2, s7, s2
-; VI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
+; VI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x41b17218, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s3, v0
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; VI-GISEL-NEXT: v_log_f32_e32 v1, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_sub_f32_e32 v0, s4, v0
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v5, s2, v1
@@ -947,32 +951,32 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s2, v1
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v1
; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; GFX900-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s7, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s7, v5
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s7, s2
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x41b17218, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s3, v0
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s4, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v1
; GFX900-GISEL-NEXT: v_fma_f32 v2, v1, v2, -v5
; GFX900-GISEL-NEXT: v_fma_f32 v2, v1, v3, v2
@@ -1260,30 +1264,30 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s0, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s3
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v1
; SI-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; SI-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; SI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v1|, v4
-; SI-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v5
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b32 s8, s8, s0
-; SI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, s8, s3
+; SI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
-; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s1, v1
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s0
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v5, s3
@@ -1294,31 +1298,31 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; SI-GISEL-NEXT: v_add_f32_e32 v6, v6, v7
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v6
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v6
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s3, s7, s3
-; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x41b17218, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: s_cselect_b32 s4, s8, s3
+; SI-GISEL-NEXT: s_cmp_lg_u32 s7, 0
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0x41b17218, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s2
; SI-GISEL-NEXT: v_log_f32_e32 v6, v0
-; SI-GISEL-NEXT: v_sub_f32_e32 v0, s8, v5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; SI-GISEL-NEXT: v_sub_f32_e32 v1, s3, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v5
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; SI-GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v6
; SI-GISEL-NEXT: v_fma_f32 v2, v6, v2, -v5
; SI-GISEL-NEXT: v_fma_f32 v2, v6, v3, v2
; SI-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v6|, v4
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v6
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v6
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s7, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
@@ -1409,42 +1413,42 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s0, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s3
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; VI-GISEL-NEXT: s_and_b32 s6, s0, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; VI-GISEL-NEXT: s_and_b32 s6, s3, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v4, s6, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v2
-; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x3805fdf4, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
-; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v7
-; VI-GISEL-NEXT: v_mul_f32_e32 v6, s6, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; VI-GISEL-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-GISEL-NEXT: v_add_f32_e32 v4, v6, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v4, v5, v4
; VI-GISEL-NEXT: v_readfirstlane_b32 s8, v4
; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x7f800000
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v1|, v4
; VI-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, s8, s0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
-; VI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; VI-GISEL-NEXT: s_cselect_b32 s6, 0x41b17218, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s1, v1
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s0
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v5, s6
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; VI-GISEL-NEXT: s_and_b32 s4, s3, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; VI-GISEL-NEXT: s_and_b32 s4, s6, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v6, s4, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x3805fdf4, v6
; VI-GISEL-NEXT: v_mul_f32_e32 v8, s4, v2
@@ -1456,19 +1460,19 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
; VI-GISEL-NEXT: v_readfirstlane_b32 s8, v6
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; VI-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; VI-GISEL-NEXT: s_cselect_b32 s4, s8, s6
; VI-GISEL-NEXT: s_cmp_lg_u32 s7, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
+; VI-GISEL-NEXT: s_cselect_b32 s5, 0x41b17218, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s7, s7, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s7
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s2
; VI-GISEL-NEXT: v_log_f32_e32 v6, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v5
-; VI-GISEL-NEXT: v_sub_f32_e32 v1, s3, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
+; VI-GISEL-NEXT: v_sub_f32_e32 v0, s3, v5
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v6
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v5, s2, v6
@@ -1480,10 +1484,10 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; VI-GISEL-NEXT: v_mul_f32_e32 v3, s2, v3
; VI-GISEL-NEXT: v_add_f32_e32 v2, v3, v2
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v6|, v4
-; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; VI-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; VI-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; VI-GISEL-NEXT: s_cselect_b32 s2, s6, s4
-; VI-GISEL-NEXT: s_cmp_lg_u32 s5, 0
+; VI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; VI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; VI-GISEL-NEXT: v_mov_b32_e32 v2, s3
; VI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
@@ -1559,65 +1563,65 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s0, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v1
; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; GFX900-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v1|, v4
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v5
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, s8, s0
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s3, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 0x41b17218, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s1, v1
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, s6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v6, 0x3f317217, v1
; GFX900-GISEL-NEXT: v_fma_f32 v7, v1, v2, -v6
; GFX900-GISEL-NEXT: v_fma_f32 v7, v1, v3, v7
; GFX900-GISEL-NEXT: v_add_f32_e32 v6, v6, v7
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v6
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, s8, s6
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s7, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s5, 0x41b17218, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s7, s7, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s7
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s2
; GFX900-GISEL-NEXT: v_log_f32_e32 v6, v0
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s6, v5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v1, s3, v1
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s3, v5
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3f317217, v6
; GFX900-GISEL-NEXT: v_fma_f32 v2, v6, v2, -v5
; GFX900-GISEL-NEXT: v_fma_f32 v2, v6, v3, v2
; GFX900-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v6|, v4
; GFX900-GISEL-NEXT: v_readfirstlane_b32 s4, v6
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s4
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s5, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
@@ -1992,29 +1996,29 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s0, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s6
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317217, v0
; SI-GISEL-NEXT: v_fma_f32 v6, v0, v3, -v2
-; SI-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; SI-GISEL-NEXT: v_fma_f32 v6, v0, v4, v6
+; SI-GISEL-NEXT: v_add_f32_e32 v2, v2, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, v5
-; SI-GISEL-NEXT: v_fma_f32 v0, v0, v4, v6
-; SI-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v2
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b32 s9, s9, s0
-; SI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
-; SI-GISEL-NEXT: s_cselect_b32 s8, 0x41b17218, 0
-; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s6, s9, s8
+; SI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; SI-GISEL-NEXT: s_cselect_b32 s7, 0x41b17218, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s1, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s0
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
@@ -2023,56 +2027,56 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; SI-GISEL-NEXT: v_fma_f32 v6, v0, v4, v6
; SI-GISEL-NEXT: v_add_f32_e32 v2, v2, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v2
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s7, s10, s7
-; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x41b17218, 0
+; SI-GISEL-NEXT: s_cselect_b32 s9, s10, s9
+; SI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; SI-GISEL-NEXT: s_cselect_b32 s8, 0x41b17218, 0
; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s4
; SI-GISEL-NEXT: v_log_f32_e32 v2, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s7
+; SI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
; SI-GISEL-NEXT: v_mul_f32_e32 v7, 0x3f317217, v2
; SI-GISEL-NEXT: v_fma_f32 v8, v2, v3, -v7
; SI-GISEL-NEXT: v_fma_f32 v8, v2, v4, v8
; SI-GISEL-NEXT: v_add_f32_e32 v7, v7, v8
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v2|, v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; SI-GISEL-NEXT: s_cmp_lg_u32 s10, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x41b17218, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s3, v1
+; SI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0x41b17218, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s2
; SI-GISEL-NEXT: v_log_f32_e32 v7, v1
-; SI-GISEL-NEXT: v_sub_f32_e32 v1, s7, v6
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s6
-; SI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s8
+; SI-GISEL-NEXT: v_sub_f32_e32 v1, s9, v6
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s5
; SI-GISEL-NEXT: v_mul_f32_e32 v6, 0x3f317217, v7
; SI-GISEL-NEXT: v_fma_f32 v3, v7, v3, -v6
; SI-GISEL-NEXT: v_fma_f32 v3, v7, v4, v3
; SI-GISEL-NEXT: v_add_f32_e32 v3, v6, v3
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v7|, v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v7
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v3
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v7
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x41b17218, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; SI-GISEL-NEXT: v_sub_f32_e32 v0, s9, v0
; SI-GISEL-NEXT: v_sub_f32_e32 v3, s2, v3
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
@@ -2179,35 +2183,35 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s6
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
-; VI-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; VI-GISEL-NEXT: s_and_b32 s6, s0, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; VI-GISEL-NEXT: s_and_b32 s6, s8, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s6, v0
; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v3
-; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x3805fdf4, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317000, v2
-; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v7
-; VI-GISEL-NEXT: v_mul_f32_e32 v6, s6, v4
+; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; VI-GISEL-NEXT: v_add_f32_e32 v2, v2, v5
+; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v4
+; VI-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; VI-GISEL-NEXT: v_mov_b32_e32 v5, 0x7f800000
-; VI-GISEL-NEXT: v_add_f32_e32 v2, v6, v2
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, v5
; VI-GISEL-NEXT: v_readfirstlane_b32 s9, v2
; VI-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, s9, s0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; VI-GISEL-NEXT: s_cselect_b32 s6, s9, s8
+; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-NEXT: s_cselect_b32 s7, 0x41b17218, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
; VI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s1, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s0
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
@@ -2229,16 +2233,16 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s5
; VI-GISEL-NEXT: v_log_f32_e32 v2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v6, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s7
; VI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; VI-GISEL-NEXT: s_and_b32 s4, s2, 0xfffff000
+; VI-GISEL-NEXT: v_mov_b32_e32 v6, s4
+; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; VI-GISEL-NEXT: s_and_b32 s4, s6, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v7, s4, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v8, 0x3805fdf4, v7
; VI-GISEL-NEXT: v_mul_f32_e32 v9, s4, v3
@@ -2248,21 +2252,21 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-GISEL-NEXT: v_mul_f32_e32 v8, s4, v4
; VI-GISEL-NEXT: v_add_f32_e32 v7, v8, v7
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v2|, v5
-; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; VI-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; VI-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; VI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; VI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x41b17218, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s3, v1
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v2, s3
; VI-GISEL-NEXT: v_log_f32_e32 v7, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; VI-GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
; VI-GISEL-NEXT: v_sub_f32_e32 v1, s9, v6
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v7
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
@@ -2369,29 +2373,29 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s6
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317217, v0
; GFX900-GISEL-NEXT: v_fma_f32 v6, v0, v3, -v2
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v6, v0, v4, v6
+; GFX900-GISEL-NEXT: v_add_f32_e32 v2, v2, v6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, v5
-; GFX900-GISEL-NEXT: v_fma_f32 v0, v0, v4, v6
-; GFX900-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s9, v0
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, s9, s0
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, s9, s8
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s7, 0x41b17218, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s8, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s1, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
@@ -2408,10 +2412,10 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s5
; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s7
; GFX900-GISEL-NEXT: v_mov_b32_e32 v6, s4
@@ -2421,23 +2425,23 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; GFX900-GISEL-NEXT: v_fma_f32 v8, v2, v4, v8
; GFX900-GISEL-NEXT: v_add_f32_e32 v7, v7, v8
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v2|, v5
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x41b17218, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x41b17218, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s3, v1
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v2, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v7, v1
; GFX900-GISEL-NEXT: v_sub_f32_e32 v1, s9, v6
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
; GFX900-GISEL-NEXT: v_mul_f32_e32 v6, 0x3f317217, v7
; GFX900-GISEL-NEXT: v_fma_f32 v3, v7, v3, -v6
; GFX900-GISEL-NEXT: v_fma_f32 v3, v7, v4, v3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 81e165d0e5972..88d8151669ea4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -45,36 +45,37 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; SI-GISEL-LABEL: s_log10_f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dword s0, s[4:5], 0xb
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s6, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s0
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
-; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
@@ -115,25 +116,25 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x369a84fb
; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3e9a2000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x369a84fb
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v3, s2, v0
; VI-GISEL-NEXT: v_mul_f32_e32 v1, s2, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v3
-; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a2000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v1, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a2000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
@@ -182,35 +183,36 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-GISEL-LABEL: s_log10_f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX900-GISEL-NEXT: s_endpgm
;
@@ -378,36 +380,37 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; SI-GISEL-LABEL: s_log10_contract_f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dword s0, s[4:5], 0xb
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s6, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s0
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
-; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v0
+; SI-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; SI-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
@@ -448,25 +451,25 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x369a84fb
; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3e9a2000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x369a84fb
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v3, s2, v0
; VI-GISEL-NEXT: v_mul_f32_e32 v1, s2, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v3
-; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a2000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v1, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a2000, v3
; VI-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-GISEL-NEXT: v_add_f32_e32 v1, v2, v1
@@ -515,35 +518,36 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; GFX900-GISEL-LABEL: s_log10_contract_f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0x3284fbcf
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v4
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e9a209a
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v1, -v3
; GFX900-GISEL-NEXT: v_fma_f32 v1, v0, v2, v1
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v3
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v0
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v0|, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX900-GISEL-NEXT: s_endpgm
;
@@ -736,42 +740,42 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s2, v1
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s4
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v1
; SI-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; SI-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; SI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
-; SI-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v5
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s7, s2
-; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x411a209b, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s3, v0
+; SI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0x411a209b, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s2
; SI-GISEL-NEXT: v_log_f32_e32 v1, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; SI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; SI-GISEL-NEXT: v_sub_f32_e32 v0, s4, v0
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v1
; SI-GISEL-NEXT: v_fma_f32 v2, v1, v2, -v5
; SI-GISEL-NEXT: v_fma_f32 v2, v1, v3, v2
; SI-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v1|, v4
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v1, s2, v1
@@ -840,38 +844,38 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s2, v1
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s2, v1
-; VI-GISEL-NEXT: s_and_b32 s4, s2, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; VI-GISEL-NEXT: s_and_b32 s4, s6, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v4, s4, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v5, s4, v2
-; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x369a84fb, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
-; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v7
-; VI-GISEL-NEXT: v_mul_f32_e32 v6, s4, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; VI-GISEL-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-GISEL-NEXT: v_add_f32_e32 v4, v6, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v5, s4, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v4, v5, v4
; VI-GISEL-NEXT: v_readfirstlane_b32 s7, v4
; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x7f800000
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; VI-GISEL-NEXT: s_cselect_b32 s2, s7, s2
-; VI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
+; VI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x411a209b, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s3, v0
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; VI-GISEL-NEXT: v_log_f32_e32 v1, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_sub_f32_e32 v0, s4, v0
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v5, s2, v1
@@ -947,32 +951,32 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s2, v1
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v1
; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; GFX900-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s7, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s7, v5
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s7, s2
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x411a209b, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s3, v0
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s4, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v1
; GFX900-GISEL-NEXT: v_fma_f32 v2, v1, v2, -v5
; GFX900-GISEL-NEXT: v_fma_f32 v2, v1, v3, v2
@@ -1260,30 +1264,30 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s0, v1
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s3
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v1
; SI-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; SI-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; SI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v1|, v4
-; SI-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; SI-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v5
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b32 s8, s8, s0
-; SI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, s8, s3
+; SI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
-; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s1, v1
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s0
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v5, s3
@@ -1294,31 +1298,31 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; SI-GISEL-NEXT: v_add_f32_e32 v6, v6, v7
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v6
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v6
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s3, s7, s3
-; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x411a209b, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: s_cselect_b32 s4, s8, s3
+; SI-GISEL-NEXT: s_cmp_lg_u32 s7, 0
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0x411a209b, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s2
; SI-GISEL-NEXT: v_log_f32_e32 v6, v0
-; SI-GISEL-NEXT: v_sub_f32_e32 v0, s8, v5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; SI-GISEL-NEXT: v_sub_f32_e32 v1, s3, v1
+; SI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v5
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; SI-GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v6
; SI-GISEL-NEXT: v_fma_f32 v2, v6, v2, -v5
; SI-GISEL-NEXT: v_fma_f32 v2, v6, v3, v2
; SI-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v6|, v4
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v6
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v6
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s7, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
; SI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
@@ -1409,42 +1413,42 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s0, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s3
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; VI-GISEL-NEXT: s_and_b32 s6, s0, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; VI-GISEL-NEXT: s_and_b32 s6, s3, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v4, s6, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v2
-; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x369a84fb, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
; VI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
-; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v7
-; VI-GISEL-NEXT: v_mul_f32_e32 v6, s6, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; VI-GISEL-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-GISEL-NEXT: v_add_f32_e32 v4, v6, v4
+; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v3
+; VI-GISEL-NEXT: v_add_f32_e32 v4, v5, v4
; VI-GISEL-NEXT: v_readfirstlane_b32 s8, v4
; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x7f800000
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v1|, v4
; VI-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, s8, s0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
-; VI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; VI-GISEL-NEXT: s_cselect_b32 s6, 0x411a209b, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s1, v1
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s0
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; VI-GISEL-NEXT: v_mov_b32_e32 v5, s6
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; VI-GISEL-NEXT: s_and_b32 s4, s3, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v1
+; VI-GISEL-NEXT: s_and_b32 s4, s6, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v6, s4, v1
; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x369a84fb, v6
; VI-GISEL-NEXT: v_mul_f32_e32 v8, s4, v2
@@ -1456,19 +1460,19 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
; VI-GISEL-NEXT: v_readfirstlane_b32 s8, v6
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; VI-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; VI-GISEL-NEXT: s_cselect_b32 s4, s8, s6
; VI-GISEL-NEXT: s_cmp_lg_u32 s7, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
+; VI-GISEL-NEXT: s_cselect_b32 s5, 0x411a209b, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s7, s7, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s7
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s2
; VI-GISEL-NEXT: v_log_f32_e32 v6, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v5
-; VI-GISEL-NEXT: v_sub_f32_e32 v1, s3, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
+; VI-GISEL-NEXT: v_sub_f32_e32 v0, s3, v5
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v6
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v5, s2, v6
@@ -1480,10 +1484,10 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-GISEL-NEXT: v_mul_f32_e32 v3, s2, v3
; VI-GISEL-NEXT: v_add_f32_e32 v2, v3, v2
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v6|, v4
-; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; VI-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; VI-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; VI-GISEL-NEXT: s_cselect_b32 s2, s6, s4
-; VI-GISEL-NEXT: s_cmp_lg_u32 s5, 0
+; VI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; VI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; VI-GISEL-NEXT: v_mov_b32_e32 v2, s3
; VI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
@@ -1559,65 +1563,65 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s0, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v1
; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v2, -v5
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX900-GISEL-NEXT: v_fma_f32 v6, v1, v3, v6
+; GFX900-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v1|, v4
-; GFX900-GISEL-NEXT: v_fma_f32 v1, v1, v3, v6
-; GFX900-GISEL-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v5
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, s8, s0
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s3, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 0x411a209b, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s1, v1
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v5, s6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-GISEL-NEXT: v_mul_f32_e32 v6, 0x3e9a209a, v1
; GFX900-GISEL-NEXT: v_fma_f32 v7, v1, v2, -v6
; GFX900-GISEL-NEXT: v_fma_f32 v7, v1, v3, v7
; GFX900-GISEL-NEXT: v_add_f32_e32 v6, v6, v7
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, v4
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v1
; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v6
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s3, s8, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, s8, s6
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s7, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s5, 0x411a209b, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s7, s7, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s7
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s2
; GFX900-GISEL-NEXT: v_log_f32_e32 v6, v0
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s6, v5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v1, s3, v1
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, s3, v5
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v1, s4, v1
; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v6
; GFX900-GISEL-NEXT: v_fma_f32 v2, v6, v2, -v5
; GFX900-GISEL-NEXT: v_fma_f32 v2, v6, v3, v2
; GFX900-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v6|, v4
; GFX900-GISEL-NEXT: v_readfirstlane_b32 s4, v6
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s4
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s5, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
@@ -1992,29 +1996,29 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s0, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s6
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v0
; SI-GISEL-NEXT: v_fma_f32 v6, v0, v3, -v2
-; SI-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; SI-GISEL-NEXT: v_fma_f32 v6, v0, v4, v6
+; SI-GISEL-NEXT: v_add_f32_e32 v2, v2, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, v5
-; SI-GISEL-NEXT: v_fma_f32 v0, v0, v4, v6
-; SI-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
-; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v2
; SI-GISEL-NEXT: s_or_b64 s[6:7], s[6:7], s[6:7]
-; SI-GISEL-NEXT: s_cselect_b32 s9, s9, s0
-; SI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
-; SI-GISEL-NEXT: s_cselect_b32 s8, 0x411a209b, 0
-; SI-GISEL-NEXT: s_or_b64 s[6:7], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s6, s9, s8
+; SI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; SI-GISEL-NEXT: s_cselect_b32 s7, 0x411a209b, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s1, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s0
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
@@ -2023,56 +2027,56 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; SI-GISEL-NEXT: v_fma_f32 v6, v0, v4, v6
; SI-GISEL-NEXT: v_add_f32_e32 v2, v2, v6
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v0
+; SI-GISEL-NEXT: v_readfirstlane_b32 s9, v0
; SI-GISEL-NEXT: v_readfirstlane_b32 s10, v2
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s7, s10, s7
-; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x411a209b, 0
+; SI-GISEL-NEXT: s_cselect_b32 s9, s10, s9
+; SI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; SI-GISEL-NEXT: s_cselect_b32 s8, 0x411a209b, 0
; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s10, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s2, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s4
; SI-GISEL-NEXT: v_log_f32_e32 v2, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s7
+; SI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
; SI-GISEL-NEXT: v_mul_f32_e32 v7, 0x3e9a209a, v2
; SI-GISEL-NEXT: v_fma_f32 v8, v2, v3, -v7
; SI-GISEL-NEXT: v_fma_f32 v8, v2, v4, v8
; SI-GISEL-NEXT: v_add_f32_e32 v7, v7, v8
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v2|, v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; SI-GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[4:5]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; SI-GISEL-NEXT: s_cmp_lg_u32 s10, 0
-; SI-GISEL-NEXT: s_cselect_b32 s6, 0x411a209b, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s3, v1
+; SI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: s_cselect_b32 s5, 0x411a209b, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v2, s2
; SI-GISEL-NEXT: v_log_f32_e32 v7, v1
-; SI-GISEL-NEXT: v_sub_f32_e32 v1, s7, v6
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s6
-; SI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; SI-GISEL-NEXT: v_mov_b32_e32 v6, s8
+; SI-GISEL-NEXT: v_sub_f32_e32 v1, s9, v6
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s5
; SI-GISEL-NEXT: v_mul_f32_e32 v6, 0x3e9a209a, v7
; SI-GISEL-NEXT: v_fma_f32 v3, v7, v3, -v6
; SI-GISEL-NEXT: v_fma_f32 v3, v7, v4, v3
; SI-GISEL-NEXT: v_add_f32_e32 v3, v6, v3
; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |v7|, v5
-; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v7
-; SI-GISEL-NEXT: v_readfirstlane_b32 s6, v3
+; SI-GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v7
+; SI-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; SI-GISEL-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-NEXT: s_cselect_b32 s2, s6, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, s5, s4
+; SI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x411a209b, 0
; SI-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; SI-GISEL-NEXT: v_sub_f32_e32 v0, s9, v0
; SI-GISEL-NEXT: v_sub_f32_e32 v3, s2, v3
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
@@ -2179,35 +2183,35 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s6
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
-; VI-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; VI-GISEL-NEXT: s_and_b32 s6, s0, 0xfffff000
+; VI-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; VI-GISEL-NEXT: s_and_b32 s6, s8, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s6, v0
; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v3
-; VI-GISEL-NEXT: v_mul_f32_e32 v7, 0x369a84fb, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a2000, v2
-; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v7
-; VI-GISEL-NEXT: v_mul_f32_e32 v6, s6, v4
+; VI-GISEL-NEXT: v_add_f32_e32 v5, v5, v6
; VI-GISEL-NEXT: v_add_f32_e32 v2, v2, v5
+; VI-GISEL-NEXT: v_mul_f32_e32 v5, s6, v4
+; VI-GISEL-NEXT: v_add_f32_e32 v2, v5, v2
; VI-GISEL-NEXT: v_mov_b32_e32 v5, 0x7f800000
-; VI-GISEL-NEXT: v_add_f32_e32 v2, v6, v2
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, v5
; VI-GISEL-NEXT: v_readfirstlane_b32 s9, v2
; VI-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, s9, s0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; VI-GISEL-NEXT: s_cselect_b32 s6, s9, s8
+; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-NEXT: s_cselect_b32 s7, 0x411a209b, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s1
; VI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s1, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s0
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
@@ -2229,16 +2233,16 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s5
; VI-GISEL-NEXT: v_log_f32_e32 v2, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v6, s4
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s7
; VI-GISEL-NEXT: v_sub_f32_e32 v0, s6, v0
-; VI-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; VI-GISEL-NEXT: s_and_b32 s4, s2, 0xfffff000
+; VI-GISEL-NEXT: v_mov_b32_e32 v6, s4
+; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; VI-GISEL-NEXT: s_and_b32 s4, s6, 0xfffff000
; VI-GISEL-NEXT: v_subrev_f32_e32 v7, s4, v2
; VI-GISEL-NEXT: v_mul_f32_e32 v8, 0x369a84fb, v7
; VI-GISEL-NEXT: v_mul_f32_e32 v9, s4, v3
@@ -2248,21 +2252,21 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-GISEL-NEXT: v_mul_f32_e32 v8, s4, v4
; VI-GISEL-NEXT: v_add_f32_e32 v7, v8, v7
; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v2|, v5
-; VI-GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; VI-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; VI-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; VI-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; VI-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
+; VI-GISEL-NEXT: s_cselect_b32 s2, 0x411a209b, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s3, v1
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v2, s3
; VI-GISEL-NEXT: v_log_f32_e32 v7, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; VI-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; VI-GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
; VI-GISEL-NEXT: v_sub_f32_e32 v1, s9, v6
; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v7
; VI-GISEL-NEXT: s_and_b32 s2, s4, 0xfffff000
@@ -2369,29 +2373,29 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s6
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v0
; GFX900-GISEL-NEXT: v_fma_f32 v6, v0, v3, -v2
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v6, v0, v4, v6
+; GFX900-GISEL-NEXT: v_add_f32_e32 v2, v2, v6
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, v5
-; GFX900-GISEL-NEXT: v_fma_f32 v0, v0, v4, v6
-; GFX900-GISEL-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s9, v0
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, s9, s0
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s6, s9, s8
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s7, 0x411a209b, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s8, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s1, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
@@ -2408,10 +2412,10 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s8, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s5
; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s7
; GFX900-GISEL-NEXT: v_mov_b32_e32 v6, s4
@@ -2421,23 +2425,23 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-GISEL-NEXT: v_fma_f32 v8, v2, v4, v8
; GFX900-GISEL-NEXT: v_add_f32_e32 v7, v7, v8
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[4:5], |v2|, v5
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v7
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s6, v2
+; GFX900-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s8, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s4, s7, s6
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v1
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x411a209b, 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x411a209b, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s6, s6, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s3, v1
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v2, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v7, v1
; GFX900-GISEL-NEXT: v_sub_f32_e32 v1, s9, v6
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, s2, v2
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, s4, v2
; GFX900-GISEL-NEXT: v_mul_f32_e32 v6, 0x3e9a209a, v7
; GFX900-GISEL-NEXT: v_fma_f32 v3, v7, v3, -v6
; GFX900-GISEL-NEXT: v_fma_f32 v3, v7, v4, v3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index fc4ac9040e7e0..b28824d9827db 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -36,17 +36,17 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
;
; SI-GISEL-LABEL: s_log2_f32:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s6, s[4:5], 0xb
+; SI-GISEL-NEXT: s_load_dword s2, s[4:5], 0xb
; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s6, v0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v1, s2
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
; SI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
; SI-GISEL-NEXT: s_cselect_b32 s3, 0x42000000, 0
@@ -84,13 +84,13 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v1, s3
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
+; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s2, v0
@@ -121,20 +121,20 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
; GFX900-GISEL: ; %bb.0:
; GFX900-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s1
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; GFX900-GISEL-NEXT: s_lshl_b32 s0, s0, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v1, s0
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
; GFX900-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v0, s2, v0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX900-GISEL-NEXT: s_endpgm
@@ -262,25 +262,25 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s2, v1
-; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
-; SI-GISEL-NEXT: s_or_b64 s[4:5], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s4
+; SI-GISEL-NEXT: s_cselect_b32 s4, 0x42000000, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v2, s3
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s3, v0
; SI-GISEL-NEXT: v_log_f32_e32 v2, v0
-; SI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
-; SI-GISEL-NEXT: v_subrev_f32_e32 v0, s2, v1
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
+; SI-GISEL-NEXT: v_subrev_f32_e32 v0, s4, v1
; SI-GISEL-NEXT: v_subrev_f32_e32 v1, s2, v2
; SI-GISEL-NEXT: s_mov_b32 s2, -1
; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
@@ -321,23 +321,23 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; VI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
+; VI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s2, v1
; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s3, v0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v2, s4
; VI-GISEL-NEXT: v_log_f32_e32 v2, v0
-; VI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; VI-GISEL-NEXT: s_cmp_lg_u32 s3, 0
; VI-GISEL-NEXT: v_subrev_f32_e32 v0, s2, v1
; VI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; VI-GISEL-NEXT: v_subrev_f32_e32 v1, s2, v2
@@ -379,23 +379,23 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s3, v0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v2, s4
; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v0
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s3, 0
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v0, s2, v1
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v1, s2, v2
@@ -560,46 +560,46 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; SI-GISEL-LABEL: s_log2_v3f32:
; SI-GISEL: ; %bb.0:
; SI-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; SI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v1
; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; SI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v1
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s8, v0
-; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s3
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
-; SI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; SI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
-; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s9, v1
; SI-GISEL-NEXT: v_subrev_f32_e32 v0, s2, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s1
-; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
-; SI-GISEL-NEXT: s_or_b64 s[0:1], vcc, vcc
-; SI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; SI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
-; SI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v2, s9, v2
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, s9
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v2, v2, s3
+; SI-GISEL-NEXT: s_cselect_b32 s4, 0x42000000, 0
+; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
+; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, s10
+; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v3, s3
; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s10, v1
; SI-GISEL-NEXT: v_log_f32_e32 v3, v1
-; SI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
-; SI-GISEL-NEXT: v_subrev_f32_e32 v1, s2, v2
-; SI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
-; SI-GISEL-NEXT: v_subrev_f32_e32 v2, s0, v3
-; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
-; SI-GISEL-NEXT: buffer_store_dword v2, off, s[4:7], 0 offset:8
+; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
+; SI-GISEL-NEXT: v_subrev_f32_e32 v1, s4, v2
+; SI-GISEL-NEXT: v_subrev_f32_e32 v2, s2, v3
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-GISEL-NEXT: buffer_store_dword v2, off, s[0:3], 0 offset:8
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: s_log2_v3f32:
@@ -640,45 +640,45 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
;
; VI-GISEL-LABEL: s_log2_v3f32:
; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; VI-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; VI-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v1
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; VI-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; VI-GISEL-NEXT: s_cmp_lg_u32 s4, 0
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s8, v0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 0x42000000, 0
-; VI-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
+; VI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s3
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
+; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
+; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s1
; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
-; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
+; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: v_subrev_f32_e32 v0, s4, v0
-; VI-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; VI-GISEL-NEXT: v_ldexp_f32 v2, v2, s1
; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; VI-GISEL-NEXT: v_ldexp_f32 v2, s9, v2
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; VI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
; VI-GISEL-NEXT: v_log_f32_e32 v2, v2
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s10, v1
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v3, s2
; VI-GISEL-NEXT: v_log_f32_e32 v3, v1
; VI-GISEL-NEXT: s_cmp_lg_u32 s1, 0
; VI-GISEL-NEXT: v_subrev_f32_e32 v1, s0, v2
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s0, v3
-; VI-GISEL-NEXT: v_mov_b32_e32 v4, s3
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s4
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, s5
; VI-GISEL-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
; VI-GISEL-NEXT: s_endpgm
;
@@ -719,45 +719,45 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
;
; GFX900-GISEL-LABEL: s_log2_v3f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX900-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX900-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX900-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], s9, v1
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s8, v0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 0x42000000, 0
-; GFX900-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX900-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s3
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v1
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
+; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX900-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s1, s1, 5
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v1
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: v_subrev_f32_e32 v0, s4, v0
-; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v2, v2, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
-; GFX900-GISEL-NEXT: v_ldexp_f32 v2, s9, v2
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX900-GISEL-NEXT: s_lshl_b32 s2, s2, 5
; GFX900-GISEL-NEXT: v_log_f32_e32 v2, v2
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s10, v1
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v3, s2
; GFX900-GISEL-NEXT: v_log_f32_e32 v3, v1
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s1, 0
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v1, s0, v2
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v2, s0, v3
; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX900-GISEL-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3]
+; GFX900-GISEL-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7]
; GFX900-GISEL-NEXT: s_endpgm
;
; GFX1100-SDAG-LABEL: s_log2_v3f32:
@@ -977,19 +977,19 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s8
; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s9, v2
-; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v0, v0, s3
; SI-GISEL-NEXT: s_cselect_b32 s4, 0x42000000, 0
; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, s9
; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v1, s9, v1
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v1, v1, s3
; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v0, s8, v0
; SI-GISEL-NEXT: s_cselect_b32 s2, 0x42000000, 0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s10, v2
; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
@@ -998,18 +998,18 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
+; SI-GISEL-NEXT: v_mov_b32_e32 v3, s10
; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s11, v2
; SI-GISEL-NEXT: v_subrev_f32_e32 v0, s4, v0
-; SI-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v3, v3, s3
; SI-GISEL-NEXT: s_cselect_b32 s4, 0x42000000, 0
; SI-GISEL-NEXT: s_or_b64 s[2:3], vcc, vcc
; SI-GISEL-NEXT: s_cselect_b32 s3, 1, 0
+; SI-GISEL-NEXT: v_mov_b32_e32 v4, s11
; SI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; SI-GISEL-NEXT: s_lshl_b32 s3, s3, 5
-; SI-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v3, s10, v3
-; SI-GISEL-NEXT: v_ldexp_f32_e32 v2, s11, v2
+; SI-GISEL-NEXT: v_ldexp_f32_e64 v2, v4, s3
; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
; SI-GISEL-NEXT: v_log_f32_e32 v4, v2
; SI-GISEL-NEXT: s_cmp_lg_u32 s2, 0
@@ -1073,23 +1073,23 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v2
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s7, s7, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s7
-; VI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
+; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v2
-; VI-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; VI-GISEL-NEXT: v_ldexp_f32 v0, v0, s6
; VI-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; VI-GISEL-NEXT: s_lshl_b32 s7, s7, 5
; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s7
-; VI-GISEL-NEXT: v_ldexp_f32 v1, s1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-NEXT: s_lshl_b32 s6, s6, 5
+; VI-GISEL-NEXT: v_ldexp_f32 v1, v1, s6
; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
-; VI-GISEL-NEXT: s_cmp_lg_u32 s6, 0
+; VI-GISEL-NEXT: s_cmp_lg_u32 s1, 0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v2
; VI-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
@@ -1100,16 +1100,16 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-GISEL-NEXT: s_lshl_b32 s1, s1, 5
; VI-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v2
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
; VI-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-NEXT: v_ldexp_f32 v3, s2, v3
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s2
; VI-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; VI-GISEL-NEXT: v_ldexp_f32 v3, v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, s3
; VI-GISEL-NEXT: s_cselect_b32 s1, 1, 0
; VI-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s2
; VI-GISEL-NEXT: v_log_f32_e32 v3, v3
-; VI-GISEL-NEXT: v_ldexp_f32 v2, s3, v2
+; VI-GISEL-NEXT: v_ldexp_f32 v2, v4, s2
; VI-GISEL-NEXT: v_log_f32_e32 v4, v2
; VI-GISEL-NEXT: s_cmp_lg_u32 s1, 0
; VI-GISEL-NEXT: v_subrev_f32_e32 v2, s0, v3
@@ -1171,23 +1171,23 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v2
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s5
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX900-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v2
-; GFX900-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: s_cselect_b32 s5, 1, 0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v0, v0, s4
; GFX900-GISEL-NEXT: s_cselect_b32 s4, 1, 0
-; GFX900-GISEL-NEXT: s_lshl_b32 s5, s5, 5
; GFX900-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; GFX900-GISEL-NEXT: v_ldexp_f32 v1, s1, v1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX900-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GFX900-GISEL-NEXT: s_lshl_b32 s4, s4, 5
+; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v1, s4
; GFX900-GISEL-NEXT: v_log_f32_e32 v1, v1
-; GFX900-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX900-GISEL-NEXT: s_cmp_lg_u32 s1, 0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v2
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
@@ -1198,16 +1198,16 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-GISEL-NEXT: s_lshl_b32 s1, s1, 5
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v2
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX900-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GFX900-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX900-GISEL-NEXT: v_ldexp_f32 v3, s2, v3
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, s2
; GFX900-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX900-GISEL-NEXT: v_ldexp_f32 v3, v3, s1
+; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, s3
; GFX900-GISEL-NEXT: s_cselect_b32 s1, 1, 0
; GFX900-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX900-GISEL-NEXT: v_log_f32_e32 v3, v3
-; GFX900-GISEL-NEXT: v_ldexp_f32 v2, s3, v2
+; GFX900-GISEL-NEXT: v_ldexp_f32 v2, v4, s2
; GFX900-GISEL-NEXT: v_log_f32_e32 v4, v2
; GFX900-GISEL-NEXT: s_cmp_lg_u32 s1, 0
; GFX900-GISEL-NEXT: v_subrev_f32_e32 v2, s0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-lds-dma-volatile-and-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-lds-dma-volatile-and-nontemporal.ll
index 3174cdf85ad1c..1963c337dff8a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-lds-dma-volatile-and-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-lds-dma-volatile-and-nontemporal.ll
@@ -29,11 +29,11 @@ define amdgpu_ps void @global_load_lds_dword_volatile(ptr addrspace(1) inreg %gp
;
; GFX942-GISEL-LABEL: global_load_lds_dword_volatile:
; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-GISEL-NEXT: s_mov_b32 m0, s2
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off sc0 sc1
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off offset:512
@@ -75,11 +75,11 @@ define amdgpu_ps void @global_load_lds_dword_nontemporal(ptr addrspace(1) inreg
;
; GFX942-GISEL-LABEL: global_load_lds_dword_nontemporal:
; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-GISEL-NEXT: s_mov_b32 m0, s2
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off nt
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off offset:512
; GFX942-GISEL-NEXT: s_endpgm
@@ -123,11 +123,11 @@ define amdgpu_ps void @global_load_lds_dword_volatile_nontemporal(ptr addrspace(
;
; GFX942-GISEL-LABEL: global_load_lds_dword_volatile_nontemporal:
; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-GISEL-NEXT: s_mov_b32 m0, s2
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off sc0 sc1
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off offset:512
@@ -173,11 +173,11 @@ define amdgpu_ps void @load_to_lds_p1_dword_volatile(ptr addrspace(1) inreg %gpt
;
; GFX942-GISEL-LABEL: load_to_lds_p1_dword_volatile:
; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-GISEL-NEXT: s_mov_b32 m0, s2
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off sc0 sc1
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off offset:512
@@ -219,11 +219,11 @@ define amdgpu_ps void @load_to_lds_p1_dword_nontemporal(ptr addrspace(1) inreg %
;
; GFX942-GISEL-LABEL: load_to_lds_p1_dword_nontemporal:
; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-GISEL-NEXT: s_mov_b32 m0, s2
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off nt
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off offset:512
; GFX942-GISEL-NEXT: s_endpgm
@@ -267,11 +267,11 @@ define amdgpu_ps void @load_to_lds_p1_dword_volatile_nontemporal(ptr addrspace(1
;
; GFX942-GISEL-LABEL: load_to_lds_p1_dword_volatile_nontemporal:
; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX942-GISEL-NEXT: s_mov_b32 m0, s2
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off sc0 sc1
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: global_load_lds_dword v[0:1], off offset:512
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 1845bb56be388..31900cd4151aa 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -83,12 +83,12 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, v0, v14
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v15, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: flat_store_dwordx4 v[14:15], v[6:9]
@@ -108,12 +108,12 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
; GFX942-GISEL-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: flat_store_byte v[4:5], v2
@@ -208,12 +208,12 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, v0, v14
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v15, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_dwordx4 v[14:15], v[6:9], off
@@ -233,12 +233,12 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
; GFX942-GISEL-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
@@ -632,12 +632,12 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0x400
; GFX942-GISEL-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, v0, v10
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v11, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
@@ -810,12 +810,12 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-GISEL-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, v0, v8
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v9, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5]
@@ -936,12 +936,12 @@ define void @memset_p1_sz1055_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0x400
; GFX942-GISEL-NEXT: .LBB6_1: ; %static-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, v0, v10
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v11, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
@@ -1069,12 +1069,12 @@ define void @memset_p1_sz2048_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-GISEL-NEXT: .LBB7_1: ; %static-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, v0, v8
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v9, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off
@@ -1720,12 +1720,12 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, s5
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, s4, v0
; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
@@ -1748,12 +1748,12 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, v0, v6
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v7, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_byte v[6:7], v2, off
@@ -1842,12 +1842,12 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, s5
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, s4, v0
; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
@@ -1870,12 +1870,12 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, s0, v0
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, v0, v6
-; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v7, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_byte v[6:7], v2, off
diff --git a/llvm/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll b/llvm/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll
index 48b09f400b2e0..f9468ef779f82 100644
--- a/llvm/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll
@@ -3,18 +3,14 @@
; Check that when mubuf addr64 instruction is handled in moveToVALU
; from the pointer, dead register writes are not emitted.
-; FIXME: We should be able to use the SGPR directly as src0 to v_add_i32
-
; GCN-LABEL: {{^}}clobber_vgpr_pair_pointer_add:
; GCN-DAG: buffer_load_dwordx2 v[[[LDPTRLO:[0-9]+]]:[[LDPTRHI:[0-9]+]]]
; GCN-DAG: s_load_dwordx2 s[[[ARG1LO:[0-9]+]]:[[ARG1HI:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0x0{{$}}
-; GCN-DAG: v_mov_b32_e32 v[[VARG1LO:[0-9]+]], s[[ARG1LO]]
; GCN-DAG: v_mov_b32_e32 v[[VARG1HI:[0-9]+]], s[[ARG1HI]]
; GCN-NOT: v_mov_b32
-; GCN-NOT: v_mov_b32
-; GCN: v_add_i32_e32 v[[PTRLO:[0-9]+]], vcc, v[[LDPTRLO]], v[[VARG1LO]]
+; GCN: v_add_i32_e32 v[[PTRLO:[0-9]+]], vcc, s[[ARG1LO]], v[[LDPTRLO]]
; GCN: v_addc_u32_e32 v[[PTRHI:[0-9]+]], vcc, v[[LDPTRHI]], v[[VARG1HI]]
; GCN: buffer_load_ubyte v{{[0-9]+}}, v[[[PTRLO]]:[[PTRHI]]],
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index d98081ad0c338..e28732cdf3aaf 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -4081,16 +4081,14 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], v[0:1]
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[2:3], v[0:1]
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_add_f32 v[2:3], s[0:1], v[0:1]
+; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s2, v3
+; GFX90A-GISEL-NEXT: v_pk_add_f32 v[2:3], v[0:1], s[2:3]
+; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s1, v2
+; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX90A-GISEL-NEXT: s_endpgm
;
@@ -4098,20 +4096,18 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], v[0:1]
+; GFX942-GISEL-NEXT: v_pk_add_f32 v[2:3], s[0:1], v[0:1]
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX942-GISEL-NEXT: v_readfirstlane_b32 s2, v3
; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[2:3], v[0:1]
+; GFX942-GISEL-NEXT: v_pk_add_f32 v[2:3], v[0:1], s[2:3]
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX942-GISEL-NEXT: v_readfirstlane_b32 s1, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX942-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index 8950163f76f04..b9e3354701b94 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -2130,58 +2130,53 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
; GFX1251-GISEL-NEXT: ds_load_2addr_b32 v[10:11], v0 offset0:6 offset1:7
; GFX1251-GISEL-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
; GFX1251-GISEL-NEXT: ds_load_2addr_b32 v[2:3], v2 offset0:4 offset1:5
-; GFX1251-GISEL-NEXT: s_wait_dscnt 0x2
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s10, v10
+; GFX1251-GISEL-NEXT: s_wait_dscnt 0x3
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s8, v8
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s9, v9
; GFX1251-GISEL-NEXT: s_wait_dscnt 0x1
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s5, v1
; GFX1251-GISEL-NEXT: s_wait_dscnt 0x0
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s6, v2
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s10, v10
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s11, v11
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s5, v5
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s6, v6
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; GFX1251-GISEL-NEXT: v_pk_max_num_f64 v[0:3], v[0:3], v[0:3]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s8, v2
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s9, v3
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s8, v8
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s9, v9
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1251-GISEL-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s3, v3
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s12, v0
-; GFX1251-GISEL-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s14, v2
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1251-GISEL-NEXT: v_xor_b32_e64 v1, 0x80000000, s2
+; GFX1251-GISEL-NEXT: v_xor_b32_e64 v3, 0x80000000, s3
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s13, v1
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s15, v3
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[12:13]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[14:15]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[4:7], v[8:11]
; GFX1251-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s6, v2
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX1251-GISEL-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1251-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index 8de7a0de592c7..df3bf2623a0e4 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -36,11 +36,11 @@ define amdgpu_cs float @v_s_exp_f32(float inreg %src) {
;
; GCN-GISEL-LABEL: v_s_exp_f32:
; GCN-GISEL: ; %bb.0:
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
-; GCN-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GCN-GISEL-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
+; GCN-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GCN-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GCN-GISEL-NEXT: s_cselect_b32 s1, 0x42800000, 0
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GCN-GISEL-NEXT: s_cselect_b32 s0, 0x42800000, 0
; GCN-GISEL-NEXT: v_add_f32_e32 v0, s0, v0
; GCN-GISEL-NEXT: v_exp_f32_e32 v0, v0
; GCN-GISEL-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
@@ -148,16 +148,16 @@ define amdgpu_cs float @v_s_log_f32(float inreg %src) {
;
; GCN-GISEL-LABEL: v_s_log_f32:
; GCN-GISEL: ; %bb.0:
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GCN-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GCN-GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
+; GCN-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v1
; GCN-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GCN-GISEL-NEXT: s_cselect_b32 s2, 1, 0
; GCN-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GCN-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GCN-GISEL-NEXT: v_ldexp_f32 v0, s0, v0
+; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GCN-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-GISEL-NEXT: s_lshl_b32 s1, s1, 5
+; GCN-GISEL-NEXT: v_ldexp_f32 v0, v0, s1
; GCN-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GCN-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GCN-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GCN-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GCN-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; GCN-GISEL-NEXT: ; return to shader part epilog
@@ -586,14 +586,13 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
; GCN-GISEL-NEXT: s_cselect_b32 s0, s2, s0
; GCN-GISEL-NEXT: v_sqrt_f32_e32 v0, s0
; GCN-GISEL-NEXT: s_cselect_b32 s1, 1, 0
+; GCN-GISEL-NEXT: v_mov_b32_e32 v1, s0
; GCN-GISEL-NEXT: v_readfirstlane_b32 s2, v0
; GCN-GISEL-NEXT: s_add_i32 s3, s2, -1
-; GCN-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GCN-GISEL-NEXT: v_fma_f32 v2, -s3, v0, v1
; GCN-GISEL-NEXT: s_add_i32 s4, s2, 1
-; GCN-GISEL-NEXT: v_fma_f32 v1, -v1, v0, s0
-; GCN-GISEL-NEXT: v_mov_b32_e32 v2, s4
-; GCN-GISEL-NEXT: v_fma_f32 v0, -v2, v0, s0
-; GCN-GISEL-NEXT: v_cmp_ge_f32_e32 vcc, 0, v1
+; GCN-GISEL-NEXT: v_fma_f32 v0, -s4, v0, v1
+; GCN-GISEL-NEXT: v_cmp_ge_f32_e32 vcc, 0, v2
; GCN-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GCN-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, 0, v0
; GCN-GISEL-NEXT: s_cselect_b32 s2, s3, s2
@@ -706,16 +705,16 @@ define amdgpu_cs float @srcmods_abs_f32(float inreg %src) {
;
; GCN-GISEL-LABEL: srcmods_abs_f32:
; GCN-GISEL: ; %bb.0:
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GCN-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], |s0|, v0
-; GCN-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GCN-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GCN-GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
+; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GCN-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], |s0|, v1
+; GCN-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
; GCN-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GCN-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GCN-GISEL-NEXT: v_ldexp_f32 v0, |s0|, v0
+; GCN-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-GISEL-NEXT: s_lshl_b32 s1, s1, 5
+; GCN-GISEL-NEXT: v_ldexp_f32 v0, |v0|, s1
; GCN-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GCN-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GCN-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GCN-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GCN-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; GCN-GISEL-NEXT: ; return to shader part epilog
@@ -764,16 +763,16 @@ define amdgpu_cs float @srcmods_neg_f32(float inreg %src) {
;
; GCN-GISEL-LABEL: srcmods_neg_f32:
; GCN-GISEL: ; %bb.0:
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, 0x800000
-; GCN-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], -s0, v0
-; GCN-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
-; GCN-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GCN-GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
+; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GCN-GISEL-NEXT: v_cmp_lt_f32_e64 s[0:1], -s0, v1
+; GCN-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
; GCN-GISEL-NEXT: s_cselect_b32 s1, 1, 0
-; GCN-GISEL-NEXT: s_lshl_b32 s2, s2, 5
-; GCN-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GCN-GISEL-NEXT: v_ldexp_f32 v0, -s0, v0
+; GCN-GISEL-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-GISEL-NEXT: s_lshl_b32 s1, s1, 5
+; GCN-GISEL-NEXT: v_ldexp_f32 v0, -v0, s1
; GCN-GISEL-NEXT: v_log_f32_e32 v0, v0
-; GCN-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GCN-GISEL-NEXT: s_cmp_lg_u32 s0, 0
; GCN-GISEL-NEXT: s_cselect_b32 s0, 0x42000000, 0
; GCN-GISEL-NEXT: v_subrev_f32_e32 v0, s0, v0
; GCN-GISEL-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f64.ll b/llvm/test/CodeGen/AMDGPU/rsq.f64.ll
index 59374d76a263f..b0cd1b8e0f9ad 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f64.ll
@@ -149,22 +149,23 @@ define amdgpu_ps <2 x i32> @s_rsq_f64(double inreg %x) {
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; SI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; SI-GISEL-CG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-CG-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; SI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v8, 0x260
-; SI-GISEL-CG-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
+; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 0xffffff80, 0
; SI-GISEL-CG-NEXT: v_cmp_class_f64_e32 vcc, v[0:1], v8
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_mul_f64 v[4:5], v[2:3], 0.5
; SI-GISEL-CG-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
; SI-GISEL-CG-NEXT: v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
; SI-GISEL-CG-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
@@ -245,13 +246,14 @@ define amdgpu_ps <2 x i32> @s_rsq_f64(double inreg %x) {
; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; VI-GISEL-CG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-CG-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; VI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
+; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; VI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; VI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
@@ -324,25 +326,24 @@ define amdgpu_ps <2 x i32> @s_rsq_f64_fabs(double inreg %x) {
;
; SI-GISEL-IR-LABEL: s_rsq_f64_fabs:
; SI-GISEL-IR: ; %bb.0:
-; SI-GISEL-IR-NEXT: v_mov_b32_e32 v0, s0
-; SI-GISEL-IR-NEXT: v_rsq_f64_e64 v[2:3], |s[0:1]|
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s2, v0
-; SI-GISEL-IR-NEXT: v_mov_b32_e32 v0, 0x260
-; SI-GISEL-IR-NEXT: v_mov_b32_e32 v1, s1
-; SI-GISEL-IR-NEXT: v_cmp_class_f64_e64 s[0:1], |s[0:1]|, v0
-; SI-GISEL-IR-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s3, v1
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s4, v2
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s5, v3
+; SI-GISEL-IR-NEXT: v_mov_b32_e32 v2, s1
+; SI-GISEL-IR-NEXT: v_rsq_f64_e64 v[0:1], |s[0:1]|
+; SI-GISEL-IR-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
+; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s3, v2
+; SI-GISEL-IR-NEXT: v_mov_b32_e32 v2, 0x260
+; SI-GISEL-IR-NEXT: s_mov_b32 s2, s0
+; SI-GISEL-IR-NEXT: v_cmp_class_f64_e64 s[0:1], |s[0:1]|, v2
+; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s4, v0
+; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s5, v1
; SI-GISEL-IR-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; SI-GISEL-IR-NEXT: s_cselect_b64 s[0:1], s[4:5], s[2:3]
-; SI-GISEL-IR-NEXT: v_mul_f64 v[0:1], s[0:1], -v[2:3]
+; SI-GISEL-IR-NEXT: v_mul_f64 v[2:3], s[0:1], -v[0:1]
; SI-GISEL-IR-NEXT: v_mov_b32_e32 v4, 0
-; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], 1.0
+; SI-GISEL-IR-NEXT: v_fma_f64 v[2:3], v[2:3], v[0:1], 1.0
; SI-GISEL-IR-NEXT: v_mov_b32_e32 v5, 0x3fd80000
-; SI-GISEL-IR-NEXT: v_mul_f64 v[6:7], v[0:1], v[2:3]
-; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], 0.5
-; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[2:3]
+; SI-GISEL-IR-NEXT: v_mul_f64 v[6:7], v[2:3], v[0:1]
+; SI-GISEL-IR-NEXT: v_fma_f64 v[2:3], v[2:3], v[4:5], 0.5
+; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[6:7], v[2:3], v[0:1]
; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s1, v1
; SI-GISEL-IR-NEXT: ; return to shader part epilog
@@ -373,11 +374,9 @@ define amdgpu_ps <2 x i32> @s_rsq_f64_fabs(double inreg %x) {
; VI-GISEL-IR-NEXT: v_rsq_f64_e64 v[0:1], |s[0:1]|
; VI-GISEL-IR-NEXT: v_mov_b32_e32 v2, 0x260
; VI-GISEL-IR-NEXT: v_cmp_class_f64_e64 s[2:3], |s[0:1]|, v2
-; VI-GISEL-IR-NEXT: v_mov_b32_e32 v3, s1
-; VI-GISEL-IR-NEXT: v_mov_b32_e32 v2, s0
-; VI-GISEL-IR-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
-; VI-GISEL-IR-NEXT: v_readfirstlane_b32 s0, v2
-; VI-GISEL-IR-NEXT: v_readfirstlane_b32 s1, v3
+; VI-GISEL-IR-NEXT: v_mov_b32_e32 v2, s1
+; VI-GISEL-IR-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2
+; VI-GISEL-IR-NEXT: v_readfirstlane_b32 s1, v2
; VI-GISEL-IR-NEXT: v_mov_b32_e32 v4, 0
; VI-GISEL-IR-NEXT: v_mov_b32_e32 v5, 0x3fd80000
; VI-GISEL-IR-NEXT: s_cmp_lg_u64 s[2:3], 0
@@ -440,22 +439,23 @@ define amdgpu_ps <2 x i32> @s_rsq_f64_fabs(double inreg %x) {
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; SI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; SI-GISEL-CG-NEXT: v_cmp_lt_f64_e64 s[2:3], |s[0:1]|, v[0:1]
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-CG-NEXT: s_or_b64 s[0:1], s[2:3], s[2:3]
+; SI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; SI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], |v[0:1]|, s1
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v8, 0x260
-; SI-GISEL-CG-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], |s[0:1]|, v0
-; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
+; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 0xffffff80, 0
; SI-GISEL-CG-NEXT: v_cmp_class_f64_e32 vcc, v[0:1], v8
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_mul_f64 v[4:5], v[2:3], 0.5
; SI-GISEL-CG-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
; SI-GISEL-CG-NEXT: v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
; SI-GISEL-CG-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
@@ -534,13 +534,14 @@ define amdgpu_ps <2 x i32> @s_rsq_f64_fabs(double inreg %x) {
; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; VI-GISEL-CG-NEXT: v_cmp_lt_f64_e64 s[2:3], |s[0:1]|, v[0:1]
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-CG-NEXT: s_cmp_lg_u64 s[2:3], 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; VI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], |s[0:1]|, v0
-; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], |v[0:1]|, s1
+; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; VI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; VI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
@@ -720,22 +721,23 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_f64(double inreg %x) {
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; SI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; SI-GISEL-CG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-CG-NEXT: s_or_b64 s[0:1], vcc, vcc
+; SI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; SI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v8, 0x260
-; SI-GISEL-CG-NEXT: s_or_b64 s[2:3], vcc, vcc
-; SI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
+; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 0xffffff80, 0
; SI-GISEL-CG-NEXT: v_cmp_class_f64_e32 vcc, v[0:1], v8
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_mul_f64 v[4:5], v[2:3], 0.5
; SI-GISEL-CG-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
; SI-GISEL-CG-NEXT: v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
; SI-GISEL-CG-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
@@ -816,13 +818,14 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_f64(double inreg %x) {
; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; VI-GISEL-CG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-CG-NEXT: s_cmp_lg_u64 vcc, 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; VI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
-; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], v[0:1], s1
+; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; VI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; VI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
@@ -895,25 +898,24 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_neg_f64(double inreg %x) {
;
; SI-GISEL-IR-LABEL: s_neg_rsq_neg_f64:
; SI-GISEL-IR: ; %bb.0:
-; SI-GISEL-IR-NEXT: v_mov_b32_e32 v0, s0
-; SI-GISEL-IR-NEXT: v_rsq_f64_e64 v[2:3], -s[0:1]
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s2, v0
-; SI-GISEL-IR-NEXT: v_mov_b32_e32 v0, 0x260
-; SI-GISEL-IR-NEXT: v_mov_b32_e32 v1, s1
-; SI-GISEL-IR-NEXT: v_cmp_class_f64_e64 s[0:1], -s[0:1], v0
-; SI-GISEL-IR-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s3, v1
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s4, v2
-; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s5, v3
+; SI-GISEL-IR-NEXT: v_mov_b32_e32 v2, s1
+; SI-GISEL-IR-NEXT: v_rsq_f64_e64 v[0:1], -s[0:1]
+; SI-GISEL-IR-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s3, v2
+; SI-GISEL-IR-NEXT: v_mov_b32_e32 v2, 0x260
+; SI-GISEL-IR-NEXT: s_mov_b32 s2, s0
+; SI-GISEL-IR-NEXT: v_cmp_class_f64_e64 s[0:1], -s[0:1], v2
+; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s4, v0
+; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s5, v1
; SI-GISEL-IR-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; SI-GISEL-IR-NEXT: s_cselect_b64 s[0:1], s[4:5], s[2:3]
-; SI-GISEL-IR-NEXT: v_mul_f64 v[0:1], s[0:1], -v[2:3]
+; SI-GISEL-IR-NEXT: v_mul_f64 v[2:3], s[0:1], -v[0:1]
; SI-GISEL-IR-NEXT: v_mov_b32_e32 v4, 0
-; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], 1.0
+; SI-GISEL-IR-NEXT: v_fma_f64 v[2:3], v[2:3], v[0:1], 1.0
; SI-GISEL-IR-NEXT: v_mov_b32_e32 v5, 0x3fd80000
-; SI-GISEL-IR-NEXT: v_mul_f64 v[6:7], v[0:1], -v[2:3]
-; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], 0.5
-; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], -v[2:3]
+; SI-GISEL-IR-NEXT: v_mul_f64 v[6:7], v[2:3], -v[0:1]
+; SI-GISEL-IR-NEXT: v_fma_f64 v[2:3], v[2:3], v[4:5], 0.5
+; SI-GISEL-IR-NEXT: v_fma_f64 v[0:1], v[6:7], v[2:3], -v[0:1]
; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-IR-NEXT: v_readfirstlane_b32 s1, v1
; SI-GISEL-IR-NEXT: ; return to shader part epilog
@@ -944,11 +946,9 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_neg_f64(double inreg %x) {
; VI-GISEL-IR-NEXT: v_rsq_f64_e64 v[0:1], -s[0:1]
; VI-GISEL-IR-NEXT: v_mov_b32_e32 v2, 0x260
; VI-GISEL-IR-NEXT: v_cmp_class_f64_e64 s[2:3], -s[0:1], v2
-; VI-GISEL-IR-NEXT: v_mov_b32_e32 v3, s1
-; VI-GISEL-IR-NEXT: v_mov_b32_e32 v2, s0
-; VI-GISEL-IR-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
-; VI-GISEL-IR-NEXT: v_readfirstlane_b32 s0, v2
-; VI-GISEL-IR-NEXT: v_readfirstlane_b32 s1, v3
+; VI-GISEL-IR-NEXT: v_mov_b32_e32 v2, s1
+; VI-GISEL-IR-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; VI-GISEL-IR-NEXT: v_readfirstlane_b32 s1, v2
; VI-GISEL-IR-NEXT: v_mov_b32_e32 v4, 0
; VI-GISEL-IR-NEXT: v_mov_b32_e32 v5, 0x3fd80000
; VI-GISEL-IR-NEXT: s_cmp_lg_u64 s[2:3], 0
@@ -1013,22 +1013,23 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_neg_f64(double inreg %x) {
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; SI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; SI-GISEL-CG-NEXT: v_cmp_lt_f64_e64 s[2:3], -s[0:1], v[0:1]
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; SI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
+; SI-GISEL-CG-NEXT: s_or_b64 s[0:1], s[2:3], s[2:3]
+; SI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; SI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], -v[0:1], s1
; SI-GISEL-CG-NEXT: v_mov_b32_e32 v8, 0x260
-; SI-GISEL-CG-NEXT: s_or_b64 s[2:3], s[2:3], s[2:3]
-; SI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; SI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; SI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; SI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], -s[0:1], v0
-; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
; SI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
+; SI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; SI-GISEL-CG-NEXT: s_cselect_b32 s2, 0xffffff80, 0
; SI-GISEL-CG-NEXT: v_cmp_class_f64_e32 vcc, v[0:1], v8
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_mul_f64 v[4:5], v[2:3], 0.5
; SI-GISEL-CG-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s0, v0
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; SI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
; SI-GISEL-CG-NEXT: v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
; SI-GISEL-CG-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
; SI-GISEL-CG-NEXT: v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
@@ -1109,13 +1110,14 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_neg_f64(double inreg %x) {
; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-GISEL-CG-NEXT: v_bfrev_b32_e32 v1, 8
; VI-GISEL-CG-NEXT: v_cmp_lt_f64_e64 s[2:3], -s[0:1], v[0:1]
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-CG-NEXT: v_mov_b32_e32 v1, s1
; VI-GISEL-CG-NEXT: s_cmp_lg_u64 s[2:3], 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s3, 1, 0
-; VI-GISEL-CG-NEXT: s_cselect_b32 s2, 1, 0
-; VI-GISEL-CG-NEXT: s_lshl_b32 s3, s3, 8
-; VI-GISEL-CG-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], -s[0:1], v0
-; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s2, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s1, 1, 0
+; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 1, 0
+; VI-GISEL-CG-NEXT: s_lshl_b32 s1, s1, 8
+; VI-GISEL-CG-NEXT: v_ldexp_f64 v[0:1], -v[0:1], s1
+; VI-GISEL-CG-NEXT: s_cmp_lg_u32 s0, 0
; VI-GISEL-CG-NEXT: s_cselect_b32 s0, 0xffffff80, 0
; VI-GISEL-CG-NEXT: v_rsq_f64_e32 v[2:3], v[0:1]
; VI-GISEL-CG-NEXT: v_readfirstlane_b32 s1, v1
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
index 10cd4fa2ab8f8..3b9da688895e4 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
@@ -68,30 +68,17 @@ define amdgpu_ps float @global_load_b32_idxprom_wrong_stride(ptr addrspace(1) al
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
;
-; GFX13-SDAG-LABEL: global_load_b32_idxprom_wrong_stride:
-; GFX13-SDAG: ; %bb.0: ; %entry
-; GFX13-SDAG-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], 3, v[0:1]
-; GFX13-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX13-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX13-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX13-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX13-GISEL-LABEL: global_load_b32_idxprom_wrong_stride:
-; GFX13-GISEL: ; %bb.0: ; %entry
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_ashrrev_i32 v1, 31, v0
-; GFX13-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 3, v[0:1]
-; GFX13-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX13-GISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX13-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX13-GISEL-NEXT: ; return to shader part epilog
+; GFX13-LABEL: global_load_b32_idxprom_wrong_stride:
+; GFX13: ; %bb.0: ; %entry
+; GFX13-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX13-NEXT: v_lshlrev_b64_e32 v[0:1], 3, v[0:1]
+; GFX13-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
+; GFX13-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
+; GFX13-NEXT: global_load_b32 v0, v[0:1], off
+; GFX13-NEXT: s_wait_loadcnt 0x0
+; GFX13-NEXT: ; return to shader part epilog
entry:
%idxprom = sext i32 %idx to i64
%arrayidx = getelementptr inbounds <2 x float>, ptr addrspace(1) %p, i64 %idxprom
@@ -617,3 +604,6 @@ entry:
}
!0 = !{i32 0, i32 1024}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX13-GISEL: {{.*}}
+; GFX13-SDAG: {{.*}}
More information about the llvm-branch-commits
mailing list