[llvm] [AMDGPU] Enable CSE in the GlobalISel post-legalizer combiner (PR #217258)
Keshav Vinayak Jha via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 03:08:26 PDT 2026
https://github.com/keshavvinayak01 created https://github.com/llvm/llvm-project/pull/217258
Enable GISel CSE in the legacy AMDGPU post-legalizer combiner so it matches the new pass-manager path.
Notify GlobalISel observers when register attributes, operands, or instruction parent blocks change, preventing stale CSE profiles. This also updates affected checks and adds a regression test for `CombinerHelper::replaceRegWith`
More context: https://github.com/llvm/llvm-project/pull/216274#pullrequestreview-4964537468
>From 98da0fd7bedb2059611e4d59928cab124ec826b0 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Wed, 19 Aug 2026 15:30:30 +0530
Subject: [PATCH] [AMDGPU] Enable CSE in the GlobalISel post-legalizer combiner
Use GISelCSEAnalysisWrapperPass in the legacy pass so it matches the CSE-enabled new pass-manager path. Notify GlobalISel observers when register attributes, operands, and instruction parent blocks change to keep CSE profiles valid.
Co-authored-by: GPT-5 <noreply at openai.com>
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 9 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 59 +++-
.../AMDGPU/AMDGPUPostLegalizerCombiner.cpp | 15 +-
...mbine-shl-from-extend-narrow.postlegal.mir | 40 +--
.../GlobalISel/crash-stack-address-O0.ll | 3 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 82 +++--
...ctlz-from-umul-to-lshr-in-postlegalizer.ll | 40 ++-
.../GlobalISel/postlegalizercombiner-ubfx.mir | 24 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 56 ++--
.../CodeGen/AMDGPU/GlobalISel/sext_inreg.ll | 22 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 32 +-
llvm/test/CodeGen/AMDGPU/div_i128.ll | 287 ++++++++----------
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 8 +-
llvm/unittests/CodeGen/GlobalISel/CSETest.cpp | 21 ++
14 files changed, 358 insertions(+), 340 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 55c4339edc39e..e8b02f3fcce07 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -189,6 +189,13 @@ bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
Register ToReg) const {
+ // constrainRegAttrs may change attributes that are part of the CSE profile
+ // for every instruction that references ToReg.
+ SmallVector<MachineInstr *, 4> ToRegInstrs;
+ for (MachineInstr &MI : MRI.reg_instructions(ToReg)) {
+ Observer.changingInstr(MI);
+ ToRegInstrs.push_back(&MI);
+ }
Observer.changingAllUsesOfReg(MRI, FromReg);
if (MRI.constrainRegAttrs(ToReg, FromReg))
@@ -197,6 +204,8 @@ void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
Builder.buildCopy(FromReg, ToReg);
Observer.finishedChangingAllUsesOfReg();
+ for (MachineInstr *MI : ToRegInstrs)
+ Observer.changedInstr(*MI);
}
void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 38b88d87051c6..6608b67363514 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -24,6 +24,8 @@
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
@@ -55,6 +57,26 @@ static cl::opt<bool> EnableNewLegality(
static constexpr unsigned MaxRegisterSize = 1024;
+// Register class is part of the CSE profile for every instruction that
+// references a register, so notify observers before changing it.
+static void setRegClassWithObserver(MachineIRBuilder &B,
+ MachineRegisterInfo &MRI, Register Reg,
+ const TargetRegisterClass *RC) {
+ GISelChangeObserver *Observer = B.getMF().getObserver();
+ if (!Observer) {
+ MRI.setRegClass(Reg, RC);
+ return;
+ }
+ SmallVector<MachineInstr *, 4> RegInstrs;
+ for (MachineInstr &MI : MRI.reg_instructions(Reg)) {
+ Observer->changingInstr(MI);
+ RegInstrs.push_back(&MI);
+ }
+ MRI.setRegClass(Reg, RC);
+ for (MachineInstr *MI : RegInstrs)
+ Observer->changedInstr(*MI);
+}
+
// Round the number of elements to the next power of two elements
static LLT getPow2VectorType(LLT Ty) {
unsigned NElts = Ty.getNumElements();
@@ -3009,8 +3031,11 @@ bool AMDGPULegalizerInfo::legalizeExtract(LegalizerHelper &Helper,
if (DstCount == 1) {
if (DstTy.isPointer())
B.buildIntToPtr(DstReg, Unmerge.getReg(StartIdx));
- else
+ else {
+ Helper.Observer.changingAllUsesOfReg(MRI, DstReg);
MRI.replaceRegWith(DstReg, Unmerge.getReg(StartIdx));
+ Helper.Observer.finishedChangingAllUsesOfReg();
+ }
} else {
SmallVector<Register, 8> MergeVec;
for (unsigned I = 0; I < DstCount; ++I)
@@ -3266,7 +3291,7 @@ bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
}
if (!B.getMRI()->getRegClassOrNull(PCReg))
- B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);
+ setRegClassWithObserver(B, *B.getMRI(), PCReg, &AMDGPU::SReg_64RegClass);
if (PtrTy.getSizeInBits() == 32)
B.buildExtract(DstReg, PCReg, 0);
@@ -3281,7 +3306,7 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
if (RequiresHighHalf && ST.has64BitLiterals()) {
if (!MRI.getRegClassOrNull(DstReg))
- MRI.setRegClass(DstReg, &AMDGPU::SReg_64RegClass);
+ setRegClassWithObserver(B, MRI, DstReg, &AMDGPU::SReg_64RegClass);
B.buildInstr(AMDGPU::S_MOV_B64)
.addDef(DstReg)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
@@ -3297,7 +3322,7 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
: MRI.createGenericVirtualRegister(I32);
if (!MRI.getRegClassOrNull(AddrLo))
- MRI.setRegClass(AddrLo, &AMDGPU::SReg_32RegClass);
+ setRegClassWithObserver(B, MRI, AddrLo, &AMDGPU::SReg_32RegClass);
// Write the lower half.
B.buildInstr(AMDGPU::S_MOV_B32)
@@ -3323,7 +3348,7 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
: MRI.createGenericVirtualRegister(LLT::integer(64));
if (!MRI.getRegClassOrNull(AddrDst))
- MRI.setRegClass(AddrDst, &AMDGPU::SReg_64RegClass);
+ setRegClassWithObserver(B, MRI, AddrDst, &AMDGPU::SReg_64RegClass);
B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
@@ -7773,7 +7798,23 @@ bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
// We need a block split to make the real endpgm a terminator. We also don't
// want to break phis in successor blocks, so we can't just delete to the
// end of the block.
+ // An instruction's parent block is part of its CSE profile, so notify
+ // observers about the instructions moved by the split.
+ GISelChangeObserver *Observer = MF->getObserver();
+ SmallVector<MachineInstr *, 8> MovedInstrs;
+ MachineBasicBlock::iterator SplitPoint(&MI);
+ ++SplitPoint;
+ if (Observer && SplitPoint != BB.end()) {
+ for (MachineInstr &MovedMI : make_range(SplitPoint, BB.end())) {
+ Observer->changingInstr(MovedMI);
+ MovedInstrs.push_back(&MovedMI);
+ }
+ }
BB.splitAt(MI, false /*UpdateLiveIns*/);
+ if (Observer) {
+ for (MachineInstr *MovedMI : MovedInstrs)
+ Observer->changedInstr(*MovedMI);
+ }
MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
MF->push_back(TrapBB);
BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
@@ -8119,7 +8160,7 @@ bool AMDGPULegalizerInfo::legalizeConstHwRegRead(MachineInstr &MI,
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
if (!MRI.getRegClassOrNull(DstReg))
- MRI.setRegClass(DstReg, &AMDGPU::SReg_32RegClass);
+ setRegClassWithObserver(B, MRI, DstReg, &AMDGPU::SReg_32RegClass);
B.buildInstr(AMDGPU::S_GETREG_B32_const)
.addDef(DstReg)
.addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
@@ -8268,8 +8309,8 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
B.buildBr(*CondBrTarget);
}
- MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
- MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
+ setRegClassWithObserver(B, MRI, Def, TRI->getWaveMaskRegClass());
+ setRegClassWithObserver(B, MRI, Use, TRI->getWaveMaskRegClass());
MI.eraseFromParent();
BrCond->eraseFromParent();
return true;
@@ -8304,7 +8345,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MI.eraseFromParent();
BrCond->eraseFromParent();
- MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
+ setRegClassWithObserver(B, MRI, Reg, TRI->getWaveMaskRegClass());
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index 66db3d49656a6..fbf6e73d75c23 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -16,6 +16,7 @@
#include "AMDGPULegalizerInfo.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/GlobalISel/Combiner.h"
#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
@@ -463,10 +464,13 @@ class AMDGPUPostLegalizerCombiner : public MachineFunctionPass {
} // end anonymous namespace
void AMDGPUPostLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<TargetPassConfig>();
AU.setPreservesCFG();
getSelectionDAGFallbackAnalysisUsage(AU);
AU.addRequired<GISelValueTrackingAnalysisLegacy>();
AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
+ AU.addRequired<GISelCSEAnalysisWrapperPass>();
+ AU.addPreserved<GISelCSEAnalysisWrapperPass>();
if (!IsOptNone) {
AU.addRequired<MachineDominatorTreeWrapperPass>();
}
@@ -482,6 +486,7 @@ AMDGPUPostLegalizerCombiner::AMDGPUPostLegalizerCombiner(bool IsOptNone)
bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
if (MF.getProperties().hasFailedISel())
return false;
+ const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
const Function &F = MF.getFunction();
bool EnableOpt =
MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
@@ -496,6 +501,10 @@ bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
IsOptNone ? nullptr
: &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ GISelCSEAnalysisWrapper &Wrapper =
+ getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
+ GISelCSEInfo *CSEInfo = &Wrapper.get(TPC.getCSEConfig());
+
CombinerInfo CInfo(/*AllowIllegalOps*/ false, /*ShouldLegalizeIllegal*/ true,
LI, EnableOpt, F.hasOptSize(), F.hasMinSize());
// Disable fixed-point iteration to reduce compile-time
@@ -503,8 +512,8 @@ bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
// Legalizer performs DCE, so a full DCE pass is unnecessary.
CInfo.EnableFullDCE = false;
- AMDGPUPostLegalizerCombinerImpl Impl(MF, CInfo, *VT, /*CSEInfo*/ nullptr,
- RuleConfig, ST, MDT, LI);
+ AMDGPUPostLegalizerCombinerImpl Impl(MF, CInfo, *VT, CSEInfo, RuleConfig, ST,
+ MDT, LI);
return Impl.combineMachineInstrs();
}
@@ -512,7 +521,9 @@ char AMDGPUPostLegalizerCombiner::ID = 0;
INITIALIZE_PASS_BEGIN(AMDGPUPostLegalizerCombiner, DEBUG_TYPE,
"Combine AMDGPU machine instrs after legalization", false,
false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
+INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
INITIALIZE_PASS_END(AMDGPUPostLegalizerCombiner, DEBUG_TYPE,
"Combine AMDGPU machine instrs after legalization", false,
false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 7e83c0d02b43f..5a337b251bab6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -115,8 +115,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -126,8 +126,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -153,8 +153,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -164,8 +164,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -191,8 +191,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -202,8 +202,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -229,8 +229,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -240,8 +240,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -348,14 +348,14 @@ body: |
; GFX6-LABEL: name: do_not_shl_s32_zero_by_16_from_zext_s16
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: %shl:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: $vgpr0 = COPY %shl(s32)
+ ; GFX6-NEXT: %extend:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: $vgpr0 = COPY %extend(s32)
;
; GFX9-LABEL: name: do_not_shl_s32_zero_by_16_from_zext_s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: %shl:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: $vgpr0 = COPY %shl(s32)
+ ; GFX9-NEXT: %extend:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: $vgpr0 = COPY %extend(s32)
%zero:_(s16) = G_CONSTANT i16 0
%extend:_(s32) = G_ZEXT %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll
index 8240c94454757..dd2b0c81727f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll
@@ -10,9 +10,8 @@ define amdgpu_kernel void @stack_write_fi() {
; CHECK-NEXT: s_add_u32 s0, s0, s17
; CHECK-NEXT: s_addc_u32 s1, s1, 0
; CHECK-NEXT: s_mov_b32 s5, 0
-; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: s_mov_b32 s4, 0
-; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; CHECK-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 5739d01e61d95..04cdd48f8f78e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -220,13 +220,12 @@ define amdgpu_ps ptr addrspace(8) @num_records_i16_raw_buffer(ptr inreg %p, i16
; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_AND_B64_]].sub0
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_1]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[S_AND_B64_]], [[S_MOV_B32_3]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[S_AND_B64_]], [[S_MOV_B32_2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_3]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
@@ -289,13 +288,12 @@ define amdgpu_ps ptr addrspace(8) @num_records_i32_raw_buffer(ptr inreg %p, i32
; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE1]].sub0
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_1]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_3]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
@@ -341,11 +339,10 @@ define amdgpu_ps float @read_raw_buffer(ptr addrspace(1) inreg %p) {
; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE1]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -462,15 +459,14 @@ define amdgpu_ps ptr addrspace(8) @variable_top_half(ptr inreg %p, i64 inreg %nu
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_4]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 16384
- ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_5]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_6]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 16384
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_5]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
@@ -541,16 +537,15 @@ define amdgpu_ps ptr addrspace(8) @general_case(ptr inreg %p, i16 inreg %stride,
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_4]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 12
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_4]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_6]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_5]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_2]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
@@ -614,16 +609,15 @@ define amdgpu_ps float @general_case_load(ptr inreg %p, i16 inreg %stride, i64 i
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_4]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 12
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_4]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_6]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_5]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_2]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
@@ -794,11 +788,10 @@ define amdgpu_ps float @read_buffer_fat_ptr_p0(ptr inreg %p) {
; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE1]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.ptr, align 1, addrspace 8)
; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -829,11 +822,10 @@ define amdgpu_ps float @read_buffer_fat_ptr_p1(ptr addrspace(1) inreg %p) {
; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE1]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.ptr, align 1, addrspace 8)
; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
index 4c0436d5f6318..99544ec35c6db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
@@ -9,8 +9,8 @@ define void @test(ptr %p) {
; CHECK-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; CHECK-NEXT: v_mov_b32_e32 v1, v2
; CHECK-NEXT: s_mov_b32 s8, 16
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_mov_b32 s11, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: s_mov_b32 s5, 0x4f7ffffe
@@ -18,50 +18,44 @@ define void @test(ptr %p) {
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s4, v2
-; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: s_mul_hi_u32 s7, s4, s7
-; CHECK-NEXT: s_add_i32 s4, s4, s7
+; CHECK-NEXT: s_mov_b32 s10, 0
+; CHECK-NEXT: s_mul_hi_u32 s6, s4, s10
+; CHECK-NEXT: s_add_i32 s4, s4, s6
; CHECK-NEXT: s_mul_hi_u32 s4, s4, s8
-; CHECK-NEXT: s_mov_b32 s7, 2
-; CHECK-NEXT: s_add_i32 s4, s4, s7
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_mov_b32 s6, 2
+; CHECK-NEXT: s_add_i32 s4, s4, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s7, v2
-; CHECK-NEXT: s_mov_b32 s9, 0
-; CHECK-NEXT: s_mul_hi_u32 s9, s7, s9
+; CHECK-NEXT: s_mul_hi_u32 s9, s7, s10
; CHECK-NEXT: s_add_i32 s7, s7, s9
; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
-; CHECK-NEXT: s_mov_b32 s9, 2
-; CHECK-NEXT: s_add_i32 s9, s7, s9
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_add_i32 s9, s7, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s7, v2
-; CHECK-NEXT: s_mov_b32 s10, 0
-; CHECK-NEXT: s_mul_hi_u32 s10, s7, s10
-; CHECK-NEXT: s_add_i32 s7, s7, s10
+; CHECK-NEXT: s_mul_hi_u32 s12, s7, s10
+; CHECK-NEXT: s_add_i32 s7, s7, s12
; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
-; CHECK-NEXT: s_mov_b32 s10, 2
-; CHECK-NEXT: s_add_i32 s7, s7, s10
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_add_i32 s7, s7, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s5, v2
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: s_mul_hi_u32 s6, s5, s6
-; CHECK-NEXT: s_add_i32 s5, s5, s6
+; CHECK-NEXT: s_mul_hi_u32 s10, s5, s10
+; CHECK-NEXT: s_add_i32 s5, s5, s10
; CHECK-NEXT: s_mul_hi_u32 s5, s5, s8
-; CHECK-NEXT: s_mov_b32 s6, 2
; CHECK-NEXT: s_add_i32 s5, s5, s6
; CHECK-NEXT: s_mov_b32 s6, 0xff
; CHECK-NEXT: s_and_b32 s4, s4, s6
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir
index 28faa0628d2e9..ba535d9bb1812 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir
@@ -15,9 +15,9 @@ body: |
; GCN: liveins: $vgpr0
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0 = COPY [[UBFX]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = G_CONSTANT i32 8
@@ -41,9 +41,9 @@ body: |
; GCN: liveins: $vgpr0_vgpr1
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0_vgpr1 = COPY [[UBFX]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 8
@@ -119,9 +119,9 @@ body: |
; GCN: liveins: $vgpr0
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0 = COPY [[UBFX]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = G_CONSTANT i32 7936 ; 31 << 8
@@ -145,9 +145,9 @@ body: |
; GCN: liveins: $vgpr0_vgpr1
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0_vgpr1 = COPY [[UBFX]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_CONSTANT i64 261888 ; 1023 << 8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 7375704ae8c42..25111293665dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -2847,47 +2847,47 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; CGP-LABEL: v_sdiv_v2i64_24bit:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v4
-; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v6
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v5
+; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v1, v5
+; CGP-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
; CGP-NEXT: v_and_b32_e32 v7, 0xffffff, v0
; CGP-NEXT: v_rcp_f32_e32 v1, v1
; CGP-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CGP-NEXT: v_cvt_u32_f32_e32 v6, v1
+; CGP-NEXT: v_cvt_u32_f32_e32 v4, v1
; CGP-NEXT: v_rcp_f32_e32 v1, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v4
-; CGP-NEXT: v_mul_lo_u32 v3, v3, v6
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v5
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v4
; CGP-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v8, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, 0, v5
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v3, 0
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, 0, v6
; CGP-NEXT: v_mul_lo_u32 v9, v0, v8
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v6, v1
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v4, v1
; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v3, 0
-; CGP-NEXT: v_and_b32_e32 v6, 0xffffff, v2
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v9, 0
-; CGP-NEXT: v_mul_lo_u32 v0, v1, v4
-; CGP-NEXT: v_add_i32_e32 v2, vcc, 1, v1
-; CGP-NEXT: v_add_i32_e64 v3, s[4:5], v8, v3
+; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v9, 0
+; CGP-NEXT: v_and_b32_e32 v9, 0xffffff, v2
+; CGP-NEXT: v_mul_lo_u32 v0, v1, v5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v1
+; CGP-NEXT: v_add_i32_e64 v4, s[4:5], v8, v4
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v7, v0
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v7, v1, v2, vcc
-; CGP-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v6, v3, 0
-; CGP-NEXT: v_sub_i32_e64 v9, s[4:5], v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; CGP-NEXT: v_mul_lo_u32 v3, v2, v5
-; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v7
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v7, v1, vcc
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v6, v3
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
+; CGP-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v9, v4, 0
+; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
+; CGP-NEXT: v_mul_lo_u32 v4, v2, v6
+; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v3
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v9, v4
; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v5
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v5
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v6
; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v5
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
index 9edfc92609ace..26ee9bc6e7879 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
@@ -1440,9 +1440,9 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
; GCN-NEXT: s_lshr_b32 s3, s1, 14
; GCN-NEXT: s_or_b32 s2, s2, s3
; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GCN-NEXT: s_mov_b32 s4, 0
; GCN-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
; GCN-NEXT: s_lshl_b32 s5, s2, 14
-; GCN-NEXT: s_mov_b32 s4, 0
; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 18
; GCN-NEXT: ; return to shader part epilog
@@ -1451,9 +1451,9 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_lshl_b64 s[2:3], s[2:3], 18
; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 14
-; GFX10PLUS-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
-; GFX10PLUS-NEXT: s_or_b32 s2, s2, s3
; GFX10PLUS-NEXT: s_mov_b32 s4, 0
+; GFX10PLUS-NEXT: s_or_b32 s2, s2, s3
+; GFX10PLUS-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX10PLUS-NEXT: s_lshl_b32 s5, s2, 14
; GFX10PLUS-NEXT: s_ashr_i64 s[2:3], s[2:3], 18
@@ -1467,10 +1467,10 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
define amdgpu_ps i65 @s_sext_inreg_i65_33(i65 inreg %value) {
; GCN-LABEL: s_sext_inreg_i65_33:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b32 s3, s2, 1
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_lshr_b64 s[4:5], s[0:1], 31
-; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GCN-NEXT: s_mov_b32 s4, 0
+; GCN-NEXT: s_lshl_b32 s5, s2, 1
+; GCN-NEXT: s_lshr_b64 s[2:3], s[0:1], 31
+; GCN-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GCN-NEXT: s_bfe_u32 s4, s0, 0x1f0000
; GCN-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
@@ -1480,10 +1480,10 @@ define amdgpu_ps i65 @s_sext_inreg_i65_33(i65 inreg %value) {
;
; GFX10PLUS-LABEL: s_sext_inreg_i65_33:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_lshl_b32 s3, s2, 1
-; GFX10PLUS-NEXT: s_mov_b32 s2, 0
-; GFX10PLUS-NEXT: s_lshr_b64 s[4:5], s[0:1], 31
-; GFX10PLUS-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10PLUS-NEXT: s_mov_b32 s4, 0
+; GFX10PLUS-NEXT: s_lshl_b32 s5, s2, 1
+; GFX10PLUS-NEXT: s_lshr_b64 s[2:3], s[0:1], 31
+; GFX10PLUS-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
; GFX10PLUS-NEXT: s_bfe_u32 s4, s0, 0x1f0000
; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX10PLUS-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index f5056d6247023..f0747d031fdc7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -1470,8 +1470,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v4, -1, v0
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x80000001, v4
; GFX6-NEXT: v_min_i32_e32 v5, -1, v0
-; GFX6-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0x80000000, v5
; GFX6-NEXT: v_max_i32_e32 v2, v4, v2
; GFX6-NEXT: v_min_i32_e32 v2, v2, v5
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
@@ -1490,8 +1489,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v4, -1, v0
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x80000001, v4
; GFX8-NEXT: v_min_i32_e32 v5, -1, v0
-; GFX8-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v6
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x80000000, v5
; GFX8-NEXT: v_max_i32_e32 v2, v4, v2
; GFX8-NEXT: v_min_i32_e32 v2, v2, v5
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v2
@@ -1586,8 +1584,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v6, -1, v0
; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000001, v6
; GFX6-NEXT: v_min_i32_e32 v8, -1, v0
-; GFX6-NEXT: v_bfrev_b32_e32 v9, 1
-; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x80000000, v8
; GFX6-NEXT: v_max_i32_e32 v3, v6, v3
; GFX6-NEXT: v_min_i32_e32 v3, v3, v8
; GFX6-NEXT: v_mov_b32_e32 v7, 0x80000001
@@ -1595,7 +1592,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v3, -1, v1
; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7
; GFX6-NEXT: v_min_i32_e32 v6, -1, v1
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v9
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v6
; GFX6-NEXT: v_max_i32_e32 v3, v3, v4
; GFX6-NEXT: v_min_i32_e32 v3, v3, v6
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
@@ -1614,8 +1611,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v6, -1, v0
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000001, v6
; GFX8-NEXT: v_min_i32_e32 v8, -1, v0
-; GFX8-NEXT: v_bfrev_b32_e32 v9, 1
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v9
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x80000000, v8
; GFX8-NEXT: v_max_i32_e32 v3, v6, v3
; GFX8-NEXT: v_min_i32_e32 v3, v3, v8
; GFX8-NEXT: v_mov_b32_e32 v7, 0x80000001
@@ -1623,7 +1619,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v3, -1, v1
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v7
; GFX8-NEXT: v_min_i32_e32 v6, -1, v1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v9
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v6
; GFX8-NEXT: v_max_i32_e32 v3, v3, v4
; GFX8-NEXT: v_min_i32_e32 v3, v3, v6
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, v1, v3
@@ -1748,14 +1744,14 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v4, -1, v1
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; GFX6-NEXT: v_min_i32_e32 v8, -1, v1
-; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x80000000, v8
; GFX6-NEXT: v_max_i32_e32 v4, v4, v5
; GFX6-NEXT: v_min_i32_e32 v4, v4, v8
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v4
; GFX6-NEXT: v_max_i32_e32 v4, -1, v2
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; GFX6-NEXT: v_min_i32_e32 v5, -1, v2
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0x80000000, v5
; GFX6-NEXT: v_max_i32_e32 v4, v4, v6
; GFX6-NEXT: v_min_i32_e32 v4, v4, v5
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
@@ -1783,14 +1779,14 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v4, -1, v1
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v9
; GFX8-NEXT: v_min_i32_e32 v8, -1, v1
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v11
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x80000000, v8
; GFX8-NEXT: v_max_i32_e32 v4, v4, v5
; GFX8-NEXT: v_min_i32_e32 v4, v4, v8
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, v1, v4
; GFX8-NEXT: v_max_i32_e32 v4, -1, v2
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v9
; GFX8-NEXT: v_min_i32_e32 v5, -1, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v11
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x80000000, v5
; GFX8-NEXT: v_max_i32_e32 v4, v4, v6
; GFX8-NEXT: v_min_i32_e32 v4, v4, v5
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v4
@@ -1943,14 +1939,14 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v5, -1, v2
; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v11
; GFX6-NEXT: v_min_i32_e32 v6, -1, v2
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v6
; GFX6-NEXT: v_max_i32_e32 v5, v5, v7
; GFX6-NEXT: v_min_i32_e32 v5, v5, v6
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
; GFX6-NEXT: v_max_i32_e32 v5, -1, v3
; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v11
; GFX6-NEXT: v_min_i32_e32 v6, -1, v3
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v6
; GFX6-NEXT: v_max_i32_e32 v5, v5, v8
; GFX6-NEXT: v_min_i32_e32 v5, v5, v6
; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v5
@@ -1985,14 +1981,14 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v5, -1, v2
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v11
; GFX8-NEXT: v_min_i32_e32 v6, -1, v2
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v13
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v6
; GFX8-NEXT: v_max_i32_e32 v5, v5, v7
; GFX8-NEXT: v_min_i32_e32 v5, v5, v6
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v5
; GFX8-NEXT: v_max_i32_e32 v5, -1, v3
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v11
; GFX8-NEXT: v_min_i32_e32 v6, -1, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v13
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v6
; GFX8-NEXT: v_max_i32_e32 v5, v5, v8
; GFX8-NEXT: v_min_i32_e32 v5, v5, v6
; GFX8-NEXT: v_sub_u32_e32 v3, vcc, v3, v5
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index b38c24f6e8eb4..0443863bc8666 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -1413,17 +1413,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v11, v2, v7
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v9, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v2, v7
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
@@ -1499,15 +1496,15 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32
; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
-; GFX9-G-O0-NEXT: s_mov_b32 s12, 64
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
+; GFX9-G-O0-NEXT: s_mov_b32 s13, 64
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s13
; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v3, v3
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v3, v3, v6
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v5, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s16, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s9
@@ -1517,28 +1514,24 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v0
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v2
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v4
; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s15, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s13, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s14, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v5, s[10:11], v3, v5
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[10:11], v3, v6, s[10:11]
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[10:11], v6, v7, s[10:11]
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[10:11], v6, v8, s[10:11]
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
@@ -1606,17 +1599,17 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 2
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 3
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_2
; GFX9-G-O0-NEXT: s_branch .LBB0_7
; GFX9-G-O0-NEXT: .LBB0_1: ; %Flow
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 4
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 5
@@ -1644,9 +1637,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_4
; GFX9-G-O0-NEXT: .LBB0_2: ; %Flow2
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 3
@@ -1718,9 +1711,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_2
; GFX9-G-O0-NEXT: .LBB0_4: ; %Flow1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 7
@@ -1749,9 +1742,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_branch .LBB0_3
; GFX9-G-O0-NEXT: .LBB0_5: ; %udiv-do-while
; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-G-O0-NEXT: v_readlane_b32 s7, v31, 9
@@ -1787,14 +1780,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], v2, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v4, v[14:15]
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
@@ -1802,8 +1795,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v24
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v25
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
@@ -1812,12 +1804,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v23
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v24
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v25
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[25:26], v0, v[2:3]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[14:15]
; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v15, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
@@ -1851,14 +1842,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[10:11], v13, v4
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v12, v9, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v7, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v6, v5, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[12:13], v13, v4
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[12:13], v12, v9, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[12:13], v10, v7, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[12:13], v6, v5, s[12:13]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v6, v12
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s9
@@ -1894,15 +1883,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
-; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s11, -1
; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s10
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[8:9], v11, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v17, s[8:9], v10, v11, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v9, v10, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v8, v9, s[8:9]
@@ -1923,9 +1909,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 9
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
@@ -1950,9 +1936,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB0_5
; GFX9-G-O0-NEXT: s_branch .LBB0_1
; GFX9-G-O0-NEXT: .LBB0_6: ; %udiv-preheader
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
@@ -2020,17 +2006,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v16
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v17
-; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s6
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
@@ -2040,9 +2023,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s8, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s9, 9
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5
@@ -2069,9 +2052,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_5
; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-bb1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
@@ -2081,21 +2064,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 1
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s7
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v2, v4
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[8:9], v2, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s6
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v7, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v1, v6, s[6:7]
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v7, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[8:9], v6, v7, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[8:9], v1, v6, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v7
@@ -2105,20 +2086,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v8, s[6:7], v1, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v8, s[8:9], v1, v2
; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
; GFX9-G-O0-NEXT: v_add_u32_e64 v2, v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_sub_u32_e64 v14, v0, v8
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
@@ -2187,9 +2167,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 6
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 7
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_4
; GFX9-G-O0-NEXT: s_branch .LBB0_6
@@ -3513,8 +3493,8 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32
; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v4, v4, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 64
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 64
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s11
; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v4, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
@@ -3523,7 +3503,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v4, v4, v6
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
-; GFX9-G-O0-NEXT: s_mov_b32 s14, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v3
@@ -3537,7 +3517,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s11
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
@@ -3546,21 +3526,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[6:7]
-; GFX9-G-O0-NEXT: s_mov_b32 s13, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v4, v5
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v5, s[6:7], v5, v6, s[6:7]
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s10
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[6:7], v6, v8, s[6:7]
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
@@ -3818,14 +3794,14 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], v2, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v4, v[20:21]
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
@@ -3833,8 +3809,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
@@ -3843,12 +3818,11 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[22:23], v0, v[2:3]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[12:13]
; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v13, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
@@ -3882,14 +3856,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[10:11], v11, v4
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v9, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[10:11], v8, v7, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v6, v5, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[12:13], v11, v4
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[12:13], v10, v9, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[12:13], v8, v7, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[12:13], v6, v5, s[12:13]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v6, v10
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s9
@@ -3933,15 +3905,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
-; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s11, -1
; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s10
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[8:9], v11, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v17, s[8:9], v10, v11, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v9, v10, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v8, v9, s[8:9]
@@ -4064,17 +4033,14 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v19
-; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s6
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
@@ -4125,21 +4091,19 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 1
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s7
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v1, v4
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[8:9], v1, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s6
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v2, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v0, v2, s[6:7]
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v3, v5, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[8:9], v2, v3, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[8:9], v0, v2, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v7
@@ -4149,18 +4113,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v3, s[6:7], v0, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v3, s[8:9], v0, v1
; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
; GFX9-G-O0-NEXT: v_add_u32_e64 v2, v3, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_sub_u32_e64 v8, v0, v3
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v3, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
@@ -4360,13 +4323,12 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v0, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v0, v[1:2]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v8
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v9
@@ -4374,28 +4336,25 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v11
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
-; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v4, v5
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v2, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v0, v2, s[6:7]
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
@@ -4473,16 +4432,14 @@ define i128 @v_sdiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
@@ -4558,13 +4515,12 @@ define i128 @v_udiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v0, v[4:5]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3
; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
@@ -4642,13 +4598,12 @@ define i128 @v_udiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v0, v[4:5]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3
; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 79e76039d6777..dbf673fa8cfee 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6939,8 +6939,8 @@ define <2 x half> @v_mul_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x) {
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -7296,8 +7296,8 @@ define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 4, 7, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -7619,8 +7619,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x h
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -7731,8 +7731,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n1_n64(<2 x i32> %arg, <2 x h
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
diff --git a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
index 6255ac3293a08..0211a230f5efb 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
@@ -9,6 +9,7 @@
#include "GISelMITest.h"
#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "gtest/gtest.h"
namespace {
@@ -668,4 +669,24 @@ TEST_F(AArch64GISelMITest, TestConstantFoldICMP) {
EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}
+TEST_F(AMDGPUGISelMITest, TestReplaceRegWithUpdatesCSE) {
+ setUp();
+ if (!TM)
+ GTEST_SKIP();
+
+ LLT S64 = LLT::scalar(64);
+ auto ZExt = B.buildZExt(S64, Copies[0]);
+ Register FromReg = MRI->createGenericVirtualRegister(S64);
+ MRI->setRegClass(FromReg, MRI->getTargetRegisterInfo()->getPointerRegClass());
+
+ GISelCSEInfo CSEInfo;
+ CSEInfo.setCSEConfig(std::make_unique<CSEConfigFull>());
+ CSEInfo.analyze(*MF);
+
+ CombinerHelper Helper(CSEInfo, B, /*IsPreLegalize=*/false);
+ Helper.replaceRegWith(*MRI, FromReg, ZExt.getReg(0));
+
+ EXPECT_FALSE(errorToBool(CSEInfo.verify()));
+}
+
} // namespace
More information about the llvm-commits
mailing list