[llvm] [AMDGPU] Enable CSE in the GlobalISel post-legalizer combiner (PR #217258)
Keshav Vinayak Jha via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 04:26:35 PDT 2026
https://github.com/keshavvinayak01 updated https://github.com/llvm/llvm-project/pull/217258
>From 98da0fd7bedb2059611e4d59928cab124ec826b0 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Wed, 19 Aug 2026 15:30:30 +0530
Subject: [PATCH 1/2] [AMDGPU] Enable CSE in the GlobalISel post-legalizer
combiner
Use GISelCSEAnalysisWrapperPass in the legacy pass so it matches the CSE-enabled new pass-manager path. Notify GlobalISel observers when register attributes, operands, and instruction parent blocks change to keep CSE profiles valid.
Co-authored-by: GPT-5 <noreply at openai.com>
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 9 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 59 +++-
.../AMDGPU/AMDGPUPostLegalizerCombiner.cpp | 15 +-
...mbine-shl-from-extend-narrow.postlegal.mir | 40 +--
.../GlobalISel/crash-stack-address-O0.ll | 3 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 82 +++--
...ctlz-from-umul-to-lshr-in-postlegalizer.ll | 40 ++-
.../GlobalISel/postlegalizercombiner-ubfx.mir | 24 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 56 ++--
.../CodeGen/AMDGPU/GlobalISel/sext_inreg.ll | 22 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 32 +-
llvm/test/CodeGen/AMDGPU/div_i128.ll | 287 ++++++++----------
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 8 +-
llvm/unittests/CodeGen/GlobalISel/CSETest.cpp | 21 ++
14 files changed, 358 insertions(+), 340 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 55c4339edc39e..e8b02f3fcce07 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -189,6 +189,13 @@ bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
Register ToReg) const {
+ // constrainRegAttrs may change attributes that are part of the CSE profile
+ // for every instruction that references ToReg.
+ SmallVector<MachineInstr *, 4> ToRegInstrs;
+ for (MachineInstr &MI : MRI.reg_instructions(ToReg)) {
+ Observer.changingInstr(MI);
+ ToRegInstrs.push_back(&MI);
+ }
Observer.changingAllUsesOfReg(MRI, FromReg);
if (MRI.constrainRegAttrs(ToReg, FromReg))
@@ -197,6 +204,8 @@ void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
Builder.buildCopy(FromReg, ToReg);
Observer.finishedChangingAllUsesOfReg();
+ for (MachineInstr *MI : ToRegInstrs)
+ Observer.changedInstr(*MI);
}
void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 38b88d87051c6..6608b67363514 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -24,6 +24,8 @@
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
@@ -55,6 +57,26 @@ static cl::opt<bool> EnableNewLegality(
static constexpr unsigned MaxRegisterSize = 1024;
+// Register class is part of the CSE profile for every instruction that
+// references a register, so notify observers before changing it.
+static void setRegClassWithObserver(MachineIRBuilder &B,
+ MachineRegisterInfo &MRI, Register Reg,
+ const TargetRegisterClass *RC) {
+ GISelChangeObserver *Observer = B.getMF().getObserver();
+ if (!Observer) {
+ MRI.setRegClass(Reg, RC);
+ return;
+ }
+ SmallVector<MachineInstr *, 4> RegInstrs;
+ for (MachineInstr &MI : MRI.reg_instructions(Reg)) {
+ Observer->changingInstr(MI);
+ RegInstrs.push_back(&MI);
+ }
+ MRI.setRegClass(Reg, RC);
+ for (MachineInstr *MI : RegInstrs)
+ Observer->changedInstr(*MI);
+}
+
// Round the number of elements to the next power of two elements
static LLT getPow2VectorType(LLT Ty) {
unsigned NElts = Ty.getNumElements();
@@ -3009,8 +3031,11 @@ bool AMDGPULegalizerInfo::legalizeExtract(LegalizerHelper &Helper,
if (DstCount == 1) {
if (DstTy.isPointer())
B.buildIntToPtr(DstReg, Unmerge.getReg(StartIdx));
- else
+ else {
+ Helper.Observer.changingAllUsesOfReg(MRI, DstReg);
MRI.replaceRegWith(DstReg, Unmerge.getReg(StartIdx));
+ Helper.Observer.finishedChangingAllUsesOfReg();
+ }
} else {
SmallVector<Register, 8> MergeVec;
for (unsigned I = 0; I < DstCount; ++I)
@@ -3266,7 +3291,7 @@ bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
}
if (!B.getMRI()->getRegClassOrNull(PCReg))
- B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);
+ setRegClassWithObserver(B, *B.getMRI(), PCReg, &AMDGPU::SReg_64RegClass);
if (PtrTy.getSizeInBits() == 32)
B.buildExtract(DstReg, PCReg, 0);
@@ -3281,7 +3306,7 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
if (RequiresHighHalf && ST.has64BitLiterals()) {
if (!MRI.getRegClassOrNull(DstReg))
- MRI.setRegClass(DstReg, &AMDGPU::SReg_64RegClass);
+ setRegClassWithObserver(B, MRI, DstReg, &AMDGPU::SReg_64RegClass);
B.buildInstr(AMDGPU::S_MOV_B64)
.addDef(DstReg)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
@@ -3297,7 +3322,7 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
: MRI.createGenericVirtualRegister(I32);
if (!MRI.getRegClassOrNull(AddrLo))
- MRI.setRegClass(AddrLo, &AMDGPU::SReg_32RegClass);
+ setRegClassWithObserver(B, MRI, AddrLo, &AMDGPU::SReg_32RegClass);
// Write the lower half.
B.buildInstr(AMDGPU::S_MOV_B32)
@@ -3323,7 +3348,7 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
: MRI.createGenericVirtualRegister(LLT::integer(64));
if (!MRI.getRegClassOrNull(AddrDst))
- MRI.setRegClass(AddrDst, &AMDGPU::SReg_64RegClass);
+ setRegClassWithObserver(B, MRI, AddrDst, &AMDGPU::SReg_64RegClass);
B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
@@ -7773,7 +7798,23 @@ bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
// We need a block split to make the real endpgm a terminator. We also don't
// want to break phis in successor blocks, so we can't just delete to the
// end of the block.
+ // An instruction's parent block is part of its CSE profile, so notify
+ // observers about the instructions moved by the split.
+ GISelChangeObserver *Observer = MF->getObserver();
+ SmallVector<MachineInstr *, 8> MovedInstrs;
+ MachineBasicBlock::iterator SplitPoint(&MI);
+ ++SplitPoint;
+ if (Observer && SplitPoint != BB.end()) {
+ for (MachineInstr &MovedMI : make_range(SplitPoint, BB.end())) {
+ Observer->changingInstr(MovedMI);
+ MovedInstrs.push_back(&MovedMI);
+ }
+ }
BB.splitAt(MI, false /*UpdateLiveIns*/);
+ if (Observer) {
+ for (MachineInstr *MovedMI : MovedInstrs)
+ Observer->changedInstr(*MovedMI);
+ }
MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
MF->push_back(TrapBB);
BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
@@ -8119,7 +8160,7 @@ bool AMDGPULegalizerInfo::legalizeConstHwRegRead(MachineInstr &MI,
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
if (!MRI.getRegClassOrNull(DstReg))
- MRI.setRegClass(DstReg, &AMDGPU::SReg_32RegClass);
+ setRegClassWithObserver(B, MRI, DstReg, &AMDGPU::SReg_32RegClass);
B.buildInstr(AMDGPU::S_GETREG_B32_const)
.addDef(DstReg)
.addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
@@ -8268,8 +8309,8 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
B.buildBr(*CondBrTarget);
}
- MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
- MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
+ setRegClassWithObserver(B, MRI, Def, TRI->getWaveMaskRegClass());
+ setRegClassWithObserver(B, MRI, Use, TRI->getWaveMaskRegClass());
MI.eraseFromParent();
BrCond->eraseFromParent();
return true;
@@ -8304,7 +8345,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MI.eraseFromParent();
BrCond->eraseFromParent();
- MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
+ setRegClassWithObserver(B, MRI, Reg, TRI->getWaveMaskRegClass());
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index 66db3d49656a6..fbf6e73d75c23 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -16,6 +16,7 @@
#include "AMDGPULegalizerInfo.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/GlobalISel/Combiner.h"
#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
@@ -463,10 +464,13 @@ class AMDGPUPostLegalizerCombiner : public MachineFunctionPass {
} // end anonymous namespace
void AMDGPUPostLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<TargetPassConfig>();
AU.setPreservesCFG();
getSelectionDAGFallbackAnalysisUsage(AU);
AU.addRequired<GISelValueTrackingAnalysisLegacy>();
AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
+ AU.addRequired<GISelCSEAnalysisWrapperPass>();
+ AU.addPreserved<GISelCSEAnalysisWrapperPass>();
if (!IsOptNone) {
AU.addRequired<MachineDominatorTreeWrapperPass>();
}
@@ -482,6 +486,7 @@ AMDGPUPostLegalizerCombiner::AMDGPUPostLegalizerCombiner(bool IsOptNone)
bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
if (MF.getProperties().hasFailedISel())
return false;
+ const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
const Function &F = MF.getFunction();
bool EnableOpt =
MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
@@ -496,6 +501,10 @@ bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
IsOptNone ? nullptr
: &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ GISelCSEAnalysisWrapper &Wrapper =
+ getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
+ GISelCSEInfo *CSEInfo = &Wrapper.get(TPC.getCSEConfig());
+
CombinerInfo CInfo(/*AllowIllegalOps*/ false, /*ShouldLegalizeIllegal*/ true,
LI, EnableOpt, F.hasOptSize(), F.hasMinSize());
// Disable fixed-point iteration to reduce compile-time
@@ -503,8 +512,8 @@ bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
// Legalizer performs DCE, so a full DCE pass is unnecessary.
CInfo.EnableFullDCE = false;
- AMDGPUPostLegalizerCombinerImpl Impl(MF, CInfo, *VT, /*CSEInfo*/ nullptr,
- RuleConfig, ST, MDT, LI);
+ AMDGPUPostLegalizerCombinerImpl Impl(MF, CInfo, *VT, CSEInfo, RuleConfig, ST,
+ MDT, LI);
return Impl.combineMachineInstrs();
}
@@ -512,7 +521,9 @@ char AMDGPUPostLegalizerCombiner::ID = 0;
INITIALIZE_PASS_BEGIN(AMDGPUPostLegalizerCombiner, DEBUG_TYPE,
"Combine AMDGPU machine instrs after legalization", false,
false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
+INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
INITIALIZE_PASS_END(AMDGPUPostLegalizerCombiner, DEBUG_TYPE,
"Combine AMDGPU machine instrs after legalization", false,
false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 7e83c0d02b43f..5a337b251bab6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -115,8 +115,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -126,8 +126,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -153,8 +153,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -164,8 +164,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -191,8 +191,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -202,8 +202,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -229,8 +229,8 @@ body: |
; GFX6-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX6-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX6-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX6-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX6-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
;
@@ -240,8 +240,8 @@ body: |
; GFX9-NEXT: %narrow:_(s32) = COPY $vgpr0
; GFX9-NEXT: %masklow30:_(s32) = G_CONSTANT i32 1073741823
; GFX9-NEXT: %masked:_(s32) = G_AND %narrow, %masklow30
- ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, [[C]](s32)
+ ; GFX9-NEXT: %shiftamt:_(s32) = G_CONSTANT i32 2
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %masked, %shiftamt(s32)
; GFX9-NEXT: %shl:_(s64) = G_ZEXT [[SHL]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(s64)
%narrow:_(s32) = COPY $vgpr0
@@ -348,14 +348,14 @@ body: |
; GFX6-LABEL: name: do_not_shl_s32_zero_by_16_from_zext_s16
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: %shl:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: $vgpr0 = COPY %shl(s32)
+ ; GFX6-NEXT: %extend:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: $vgpr0 = COPY %extend(s32)
;
; GFX9-LABEL: name: do_not_shl_s32_zero_by_16_from_zext_s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: %shl:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: $vgpr0 = COPY %shl(s32)
+ ; GFX9-NEXT: %extend:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: $vgpr0 = COPY %extend(s32)
%zero:_(s16) = G_CONSTANT i16 0
%extend:_(s32) = G_ZEXT %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll
index 8240c94454757..dd2b0c81727f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll
@@ -10,9 +10,8 @@ define amdgpu_kernel void @stack_write_fi() {
; CHECK-NEXT: s_add_u32 s0, s0, s17
; CHECK-NEXT: s_addc_u32 s1, s1, 0
; CHECK-NEXT: s_mov_b32 s5, 0
-; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: s_mov_b32 s4, 0
-; CHECK-NEXT: v_mov_b32_e32 v0, s6
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; CHECK-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 5739d01e61d95..04cdd48f8f78e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -220,13 +220,12 @@ define amdgpu_ps ptr addrspace(8) @num_records_i16_raw_buffer(ptr inreg %p, i16
; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_AND_B64_]].sub0
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_1]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[S_AND_B64_]], [[S_MOV_B32_3]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[S_AND_B64_]], [[S_MOV_B32_2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_3]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
@@ -289,13 +288,12 @@ define amdgpu_ps ptr addrspace(8) @num_records_i32_raw_buffer(ptr inreg %p, i32
; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE1]].sub0
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_1]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_3]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
@@ -341,11 +339,10 @@ define amdgpu_ps float @read_raw_buffer(ptr addrspace(1) inreg %p) {
; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE1]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -462,15 +459,14 @@ define amdgpu_ps ptr addrspace(8) @variable_top_half(ptr inreg %p, i64 inreg %nu
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_4]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 16384
- ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_5]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_6]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 16384
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_5]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
@@ -541,16 +537,15 @@ define amdgpu_ps ptr addrspace(8) @general_case(ptr inreg %p, i16 inreg %stride,
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_4]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 12
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_4]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_6]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_5]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_2]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
@@ -614,16 +609,15 @@ define amdgpu_ps float @general_case_load(ptr inreg %p, i16 inreg %stride, i64 i
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_4]], implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 12
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_4]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_6]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_5]], implicit-def dead $scc
; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_LSHL_B32_2]], %subreg.sub1
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
@@ -794,11 +788,10 @@ define amdgpu_ps float @read_buffer_fat_ptr_p0(ptr inreg %p) {
; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE1]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.ptr, align 1, addrspace 8)
; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -829,11 +822,10 @@ define amdgpu_ps float @read_buffer_fat_ptr_p1(ptr addrspace(1) inreg %p) {
; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE1]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.ptr, align 1, addrspace 8)
; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
index 4c0436d5f6318..99544ec35c6db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
@@ -9,8 +9,8 @@ define void @test(ptr %p) {
; CHECK-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; CHECK-NEXT: v_mov_b32_e32 v1, v2
; CHECK-NEXT: s_mov_b32 s8, 16
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_mov_b32 s11, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: s_mov_b32 s5, 0x4f7ffffe
@@ -18,50 +18,44 @@ define void @test(ptr %p) {
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s4, v2
-; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: s_mul_hi_u32 s7, s4, s7
-; CHECK-NEXT: s_add_i32 s4, s4, s7
+; CHECK-NEXT: s_mov_b32 s10, 0
+; CHECK-NEXT: s_mul_hi_u32 s6, s4, s10
+; CHECK-NEXT: s_add_i32 s4, s4, s6
; CHECK-NEXT: s_mul_hi_u32 s4, s4, s8
-; CHECK-NEXT: s_mov_b32 s7, 2
-; CHECK-NEXT: s_add_i32 s4, s4, s7
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_mov_b32 s6, 2
+; CHECK-NEXT: s_add_i32 s4, s4, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s7, v2
-; CHECK-NEXT: s_mov_b32 s9, 0
-; CHECK-NEXT: s_mul_hi_u32 s9, s7, s9
+; CHECK-NEXT: s_mul_hi_u32 s9, s7, s10
; CHECK-NEXT: s_add_i32 s7, s7, s9
; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
-; CHECK-NEXT: s_mov_b32 s9, 2
-; CHECK-NEXT: s_add_i32 s9, s7, s9
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_add_i32 s9, s7, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s7, v2
-; CHECK-NEXT: s_mov_b32 s10, 0
-; CHECK-NEXT: s_mul_hi_u32 s10, s7, s10
-; CHECK-NEXT: s_add_i32 s7, s7, s10
+; CHECK-NEXT: s_mul_hi_u32 s12, s7, s10
+; CHECK-NEXT: s_add_i32 s7, s7, s12
; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
-; CHECK-NEXT: s_mov_b32 s10, 2
-; CHECK-NEXT: s_add_i32 s7, s7, s10
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_add_i32 s7, s7, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s11
; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s5, v2
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: s_mul_hi_u32 s6, s5, s6
-; CHECK-NEXT: s_add_i32 s5, s5, s6
+; CHECK-NEXT: s_mul_hi_u32 s10, s5, s10
+; CHECK-NEXT: s_add_i32 s5, s5, s10
; CHECK-NEXT: s_mul_hi_u32 s5, s5, s8
-; CHECK-NEXT: s_mov_b32 s6, 2
; CHECK-NEXT: s_add_i32 s5, s5, s6
; CHECK-NEXT: s_mov_b32 s6, 0xff
; CHECK-NEXT: s_and_b32 s4, s4, s6
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir
index 28faa0628d2e9..ba535d9bb1812 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ubfx.mir
@@ -15,9 +15,9 @@ body: |
; GCN: liveins: $vgpr0
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0 = COPY [[UBFX]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = G_CONSTANT i32 8
@@ -41,9 +41,9 @@ body: |
; GCN: liveins: $vgpr0_vgpr1
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0_vgpr1 = COPY [[UBFX]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 8
@@ -119,9 +119,9 @@ body: |
; GCN: liveins: $vgpr0
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s32) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0 = COPY [[UBFX]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = G_CONSTANT i32 7936 ; 31 << 8
@@ -145,9 +145,9 @@ body: |
; GCN: liveins: $vgpr0_vgpr1
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C1]](s32), [[C]]
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+ ; GCN-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+ ; GCN-NEXT: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY]], [[C]](s32), [[C1]]
; GCN-NEXT: $vgpr0_vgpr1 = COPY [[UBFX]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = G_CONSTANT i64 261888 ; 1023 << 8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 7375704ae8c42..25111293665dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -2847,47 +2847,47 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; CGP-LABEL: v_sdiv_v2i64_24bit:
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; CGP-NEXT: v_cvt_f32_u32_e32 v1, v4
-; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v6
-; CGP-NEXT: v_cvt_f32_u32_e32 v3, v5
+; CGP-NEXT: v_and_b32_e32 v5, 0xffffff, v4
+; CGP-NEXT: v_cvt_f32_u32_e32 v1, v5
+; CGP-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; CGP-NEXT: v_cvt_f32_u32_e32 v3, v6
; CGP-NEXT: v_and_b32_e32 v7, 0xffffff, v0
; CGP-NEXT: v_rcp_f32_e32 v1, v1
; CGP-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CGP-NEXT: v_cvt_u32_f32_e32 v6, v1
+; CGP-NEXT: v_cvt_u32_f32_e32 v4, v1
; CGP-NEXT: v_rcp_f32_e32 v1, v3
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v4
-; CGP-NEXT: v_mul_lo_u32 v3, v3, v6
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, 0, v5
+; CGP-NEXT: v_mul_lo_u32 v3, v3, v4
; CGP-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1
; CGP-NEXT: v_cvt_u32_f32_e32 v8, v0
-; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, 0, v5
+; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v3, 0
+; CGP-NEXT: v_sub_i32_e32 v0, vcc, 0, v6
; CGP-NEXT: v_mul_lo_u32 v9, v0, v8
-; CGP-NEXT: v_add_i32_e32 v3, vcc, v6, v1
+; CGP-NEXT: v_add_i32_e32 v3, vcc, v4, v1
; CGP-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v3, 0
-; CGP-NEXT: v_and_b32_e32 v6, 0xffffff, v2
-; CGP-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v9, 0
-; CGP-NEXT: v_mul_lo_u32 v0, v1, v4
-; CGP-NEXT: v_add_i32_e32 v2, vcc, 1, v1
-; CGP-NEXT: v_add_i32_e64 v3, s[4:5], v8, v3
+; CGP-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v9, 0
+; CGP-NEXT: v_and_b32_e32 v9, 0xffffff, v2
+; CGP-NEXT: v_mul_lo_u32 v0, v1, v5
+; CGP-NEXT: v_add_i32_e32 v3, vcc, 1, v1
+; CGP-NEXT: v_add_i32_e64 v4, s[4:5], v8, v4
; CGP-NEXT: v_sub_i32_e32 v0, vcc, v7, v0
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v7, v1, v2, vcc
-; CGP-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v6, v3, 0
-; CGP-NEXT: v_sub_i32_e64 v9, s[4:5], v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; CGP-NEXT: v_mul_lo_u32 v3, v2, v5
-; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v7
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT: v_cndmask_b32_e32 v0, v7, v1, vcc
-; CGP-NEXT: v_sub_i32_e32 v3, vcc, v6, v3
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc
+; CGP-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v9, v4, 0
+; CGP-NEXT: v_sub_i32_e64 v7, s[4:5], v0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
+; CGP-NEXT: v_mul_lo_u32 v4, v2, v6
+; CGP-NEXT: v_add_i32_e32 v1, vcc, 1, v3
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5
+; CGP-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
+; CGP-NEXT: v_sub_i32_e32 v3, vcc, v9, v4
; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v5
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v5
+; CGP-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v6
; CGP-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; CGP-NEXT: v_add_i32_e32 v4, vcc, 1, v2
-; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v5
+; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v3, v6
; CGP-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; CGP-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; CGP-NEXT: v_ashrrev_i32_e32 v3, 31, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
index 9edfc92609ace..26ee9bc6e7879 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
@@ -1440,9 +1440,9 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
; GCN-NEXT: s_lshr_b32 s3, s1, 14
; GCN-NEXT: s_or_b32 s2, s2, s3
; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GCN-NEXT: s_mov_b32 s4, 0
; GCN-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
; GCN-NEXT: s_lshl_b32 s5, s2, 14
-; GCN-NEXT: s_mov_b32 s4, 0
; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 18
; GCN-NEXT: ; return to shader part epilog
@@ -1451,9 +1451,9 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_lshl_b64 s[2:3], s[2:3], 18
; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 14
-; GFX10PLUS-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
-; GFX10PLUS-NEXT: s_or_b32 s2, s2, s3
; GFX10PLUS-NEXT: s_mov_b32 s4, 0
+; GFX10PLUS-NEXT: s_or_b32 s2, s2, s3
+; GFX10PLUS-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX10PLUS-NEXT: s_lshl_b32 s5, s2, 14
; GFX10PLUS-NEXT: s_ashr_i64 s[2:3], s[2:3], 18
@@ -1467,10 +1467,10 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
define amdgpu_ps i65 @s_sext_inreg_i65_33(i65 inreg %value) {
; GCN-LABEL: s_sext_inreg_i65_33:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b32 s3, s2, 1
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_lshr_b64 s[4:5], s[0:1], 31
-; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GCN-NEXT: s_mov_b32 s4, 0
+; GCN-NEXT: s_lshl_b32 s5, s2, 1
+; GCN-NEXT: s_lshr_b64 s[2:3], s[0:1], 31
+; GCN-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GCN-NEXT: s_bfe_u32 s4, s0, 0x1f0000
; GCN-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
@@ -1480,10 +1480,10 @@ define amdgpu_ps i65 @s_sext_inreg_i65_33(i65 inreg %value) {
;
; GFX10PLUS-LABEL: s_sext_inreg_i65_33:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_lshl_b32 s3, s2, 1
-; GFX10PLUS-NEXT: s_mov_b32 s2, 0
-; GFX10PLUS-NEXT: s_lshr_b64 s[4:5], s[0:1], 31
-; GFX10PLUS-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10PLUS-NEXT: s_mov_b32 s4, 0
+; GFX10PLUS-NEXT: s_lshl_b32 s5, s2, 1
+; GFX10PLUS-NEXT: s_lshr_b64 s[2:3], s[0:1], 31
+; GFX10PLUS-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
; GFX10PLUS-NEXT: s_bfe_u32 s4, s0, 0x1f0000
; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX10PLUS-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index f5056d6247023..f0747d031fdc7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -1470,8 +1470,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v4, -1, v0
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x80000001, v4
; GFX6-NEXT: v_min_i32_e32 v5, -1, v0
-; GFX6-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0x80000000, v5
; GFX6-NEXT: v_max_i32_e32 v2, v4, v2
; GFX6-NEXT: v_min_i32_e32 v2, v2, v5
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
@@ -1490,8 +1489,7 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v4, -1, v0
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x80000001, v4
; GFX8-NEXT: v_min_i32_e32 v5, -1, v0
-; GFX8-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v6
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x80000000, v5
; GFX8-NEXT: v_max_i32_e32 v2, v4, v2
; GFX8-NEXT: v_min_i32_e32 v2, v2, v5
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v0, v2
@@ -1586,8 +1584,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v6, -1, v0
; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000001, v6
; GFX6-NEXT: v_min_i32_e32 v8, -1, v0
-; GFX6-NEXT: v_bfrev_b32_e32 v9, 1
-; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x80000000, v8
; GFX6-NEXT: v_max_i32_e32 v3, v6, v3
; GFX6-NEXT: v_min_i32_e32 v3, v3, v8
; GFX6-NEXT: v_mov_b32_e32 v7, 0x80000001
@@ -1595,7 +1592,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v3, -1, v1
; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7
; GFX6-NEXT: v_min_i32_e32 v6, -1, v1
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v9
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v6
; GFX6-NEXT: v_max_i32_e32 v3, v3, v4
; GFX6-NEXT: v_min_i32_e32 v3, v3, v6
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
@@ -1614,8 +1611,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v6, -1, v0
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000001, v6
; GFX8-NEXT: v_min_i32_e32 v8, -1, v0
-; GFX8-NEXT: v_bfrev_b32_e32 v9, 1
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v9
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x80000000, v8
; GFX8-NEXT: v_max_i32_e32 v3, v6, v3
; GFX8-NEXT: v_min_i32_e32 v3, v3, v8
; GFX8-NEXT: v_mov_b32_e32 v7, 0x80000001
@@ -1623,7 +1619,7 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v3, -1, v1
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v7
; GFX8-NEXT: v_min_i32_e32 v6, -1, v1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v9
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v6
; GFX8-NEXT: v_max_i32_e32 v3, v3, v4
; GFX8-NEXT: v_min_i32_e32 v3, v3, v6
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, v1, v3
@@ -1748,14 +1744,14 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v4, -1, v1
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; GFX6-NEXT: v_min_i32_e32 v8, -1, v1
-; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x80000000, v8
; GFX6-NEXT: v_max_i32_e32 v4, v4, v5
; GFX6-NEXT: v_min_i32_e32 v4, v4, v8
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v4
; GFX6-NEXT: v_max_i32_e32 v4, -1, v2
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v9
; GFX6-NEXT: v_min_i32_e32 v5, -1, v2
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0x80000000, v5
; GFX6-NEXT: v_max_i32_e32 v4, v4, v6
; GFX6-NEXT: v_min_i32_e32 v4, v4, v5
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
@@ -1783,14 +1779,14 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v4, -1, v1
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v9
; GFX8-NEXT: v_min_i32_e32 v8, -1, v1
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v11
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x80000000, v8
; GFX8-NEXT: v_max_i32_e32 v4, v4, v5
; GFX8-NEXT: v_min_i32_e32 v4, v4, v8
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, v1, v4
; GFX8-NEXT: v_max_i32_e32 v4, -1, v2
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v9
; GFX8-NEXT: v_min_i32_e32 v5, -1, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v11
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x80000000, v5
; GFX8-NEXT: v_max_i32_e32 v4, v4, v6
; GFX8-NEXT: v_min_i32_e32 v4, v4, v5
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v4
@@ -1943,14 +1939,14 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
; GFX6-NEXT: v_max_i32_e32 v5, -1, v2
; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v11
; GFX6-NEXT: v_min_i32_e32 v6, -1, v2
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v6
; GFX6-NEXT: v_max_i32_e32 v5, v5, v7
; GFX6-NEXT: v_min_i32_e32 v5, v5, v6
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
; GFX6-NEXT: v_max_i32_e32 v5, -1, v3
; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v11
; GFX6-NEXT: v_min_i32_e32 v6, -1, v3
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0x80000000, v6
; GFX6-NEXT: v_max_i32_e32 v5, v5, v8
; GFX6-NEXT: v_min_i32_e32 v5, v5, v6
; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v5
@@ -1985,14 +1981,14 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
; GFX8-NEXT: v_max_i32_e32 v5, -1, v2
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v11
; GFX8-NEXT: v_min_i32_e32 v6, -1, v2
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v13
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v6
; GFX8-NEXT: v_max_i32_e32 v5, v5, v7
; GFX8-NEXT: v_min_i32_e32 v5, v5, v6
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v5
; GFX8-NEXT: v_max_i32_e32 v5, -1, v3
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v11
; GFX8-NEXT: v_min_i32_e32 v6, -1, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v13
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x80000000, v6
; GFX8-NEXT: v_max_i32_e32 v5, v5, v8
; GFX8-NEXT: v_min_i32_e32 v5, v5, v6
; GFX8-NEXT: v_sub_u32_e32 v3, vcc, v3, v5
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index b38c24f6e8eb4..0443863bc8666 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -1413,17 +1413,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v11, v2, v7
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v9, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v2, v7
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
@@ -1499,15 +1496,15 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32
; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
-; GFX9-G-O0-NEXT: s_mov_b32 s12, 64
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
+; GFX9-G-O0-NEXT: s_mov_b32 s13, 64
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s13
; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v3, v3
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v3, v3, v6
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v5, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s16, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s9
@@ -1517,28 +1514,24 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v0
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v2
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v4
; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s15, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s13, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s14, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v5, s[10:11], v3, v5
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[10:11], v3, v6, s[10:11]
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[10:11], v6, v7, s[10:11]
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[10:11], v6, v8, s[10:11]
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
@@ -1606,17 +1599,17 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 2
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 3
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_2
; GFX9-G-O0-NEXT: s_branch .LBB0_7
; GFX9-G-O0-NEXT: .LBB0_1: ; %Flow
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 4
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 5
@@ -1644,9 +1637,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_4
; GFX9-G-O0-NEXT: .LBB0_2: ; %Flow2
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 3
@@ -1718,9 +1711,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_2
; GFX9-G-O0-NEXT: .LBB0_4: ; %Flow1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 7
@@ -1749,9 +1742,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_branch .LBB0_3
; GFX9-G-O0-NEXT: .LBB0_5: ; %udiv-do-while
; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-G-O0-NEXT: v_readlane_b32 s7, v31, 9
@@ -1787,14 +1780,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], v2, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v4, v[14:15]
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
@@ -1802,8 +1795,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v24
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v25
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
@@ -1812,12 +1804,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v23
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v24
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v25
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[25:26], v0, v[2:3]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[14:15]
; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v15, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
@@ -1851,14 +1842,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[10:11], v13, v4
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v12, v9, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v7, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v6, v5, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[12:13], v13, v4
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[12:13], v12, v9, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[12:13], v10, v7, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[12:13], v6, v5, s[12:13]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v6, v12
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s9
@@ -1894,15 +1883,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
-; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s11, -1
; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s10
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[8:9], v11, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v17, s[8:9], v10, v11, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v9, v10, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v8, v9, s[8:9]
@@ -1923,9 +1909,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 9
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
@@ -1950,9 +1936,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB0_5
; GFX9-G-O0-NEXT: s_branch .LBB0_1
; GFX9-G-O0-NEXT: .LBB0_6: ; %udiv-preheader
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
@@ -2020,17 +2006,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v16
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v17
-; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s6
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
@@ -2040,9 +2023,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s8, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s9, 9
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5
@@ -2069,9 +2052,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_5
; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-bb1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
@@ -2081,21 +2064,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 1
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s7
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v2, v4
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[8:9], v2, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s6
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v7, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v1, v6, s[6:7]
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v5, v7, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[8:9], v6, v7, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[8:9], v1, v6, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v7
@@ -2105,20 +2086,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v8, s[6:7], v1, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v8, s[8:9], v1, v2
; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
; GFX9-G-O0-NEXT: v_add_u32_e64 v2, v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_sub_u32_e64 v14, v0, v8
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
@@ -2187,9 +2167,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 6
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 7
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_4
; GFX9-G-O0-NEXT: s_branch .LBB0_6
@@ -3513,8 +3493,8 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32
; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v4, v4, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 64
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 64
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s11
; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v4, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
@@ -3523,7 +3503,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v4, v4, v6
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
-; GFX9-G-O0-NEXT: s_mov_b32 s14, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v3
@@ -3537,7 +3517,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s11
; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
@@ -3546,21 +3526,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[6:7]
-; GFX9-G-O0-NEXT: s_mov_b32 s13, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v4, v5
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v5, s[6:7], v5, v6, s[6:7]
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s10
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[6:7], v6, v8, s[6:7]
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
@@ -3818,14 +3794,14 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], v2, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v4, v[20:21]
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
@@ -3833,8 +3809,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
@@ -3843,12 +3818,11 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[22:23], v0, v[2:3]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[12:13]
; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v13, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
@@ -3882,14 +3856,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[10:11], v11, v4
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v9, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[10:11], v8, v7, s[10:11]
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v6, v5, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[12:13], v11, v4
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[12:13], v10, v9, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[12:13], v8, v7, s[12:13]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[12:13], v6, v5, s[12:13]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v6, v10
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s9
@@ -3933,15 +3905,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
-; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s11, -1
; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s10
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[8:9], v11, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v17, s[8:9], v10, v11, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v9, v10, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v8, v9, s[8:9]
@@ -4064,17 +4033,14 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v19
-; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
-; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s6
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
@@ -4125,21 +4091,19 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 1
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s7
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v1, v4
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[8:9], v1, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s6
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v2, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v0, v2, s[6:7]
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[8:9], v3, v5, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[8:9], v2, v3, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[8:9], v0, v2, s[8:9]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v7
@@ -4149,18 +4113,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v3, s[6:7], v0, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v3, s[8:9], v0, v1
; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
; GFX9-G-O0-NEXT: v_add_u32_e64 v2, v3, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_sub_u32_e64 v8, v0, v3
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v3, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
@@ -4360,13 +4323,12 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v0, v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v0, v[1:2]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v8
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v9
@@ -4374,28 +4336,25 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v11
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
-; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v4, v5
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v2, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v0, v2, s[6:7]
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
@@ -4473,16 +4432,14 @@ define i128 @v_sdiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
@@ -4558,13 +4515,12 @@ define i128 @v_udiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v0, v[4:5]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3
; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
@@ -4642,13 +4598,12 @@ define i128 @v_udiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v0, v[4:5]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3
; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 79e76039d6777..dbf673fa8cfee 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6939,8 +6939,8 @@ define <2 x half> @v_mul_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x) {
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -7296,8 +7296,8 @@ define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 4, 7, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -7619,8 +7619,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x h
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 6, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -7731,8 +7731,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n1_n64(<2 x i32> %arg, <2 x h
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 6, 0, vcc
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc
+; GFX9-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX9-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
diff --git a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
index 6255ac3293a08..0211a230f5efb 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
@@ -9,6 +9,7 @@
#include "GISelMITest.h"
#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "gtest/gtest.h"
namespace {
@@ -668,4 +669,24 @@ TEST_F(AArch64GISelMITest, TestConstantFoldICMP) {
EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}
+TEST_F(AMDGPUGISelMITest, TestReplaceRegWithUpdatesCSE) {
+ setUp();
+ if (!TM)
+ GTEST_SKIP();
+
+ LLT S64 = LLT::scalar(64);
+ auto ZExt = B.buildZExt(S64, Copies[0]);
+ Register FromReg = MRI->createGenericVirtualRegister(S64);
+ MRI->setRegClass(FromReg, MRI->getTargetRegisterInfo()->getPointerRegClass());
+
+ GISelCSEInfo CSEInfo;
+ CSEInfo.setCSEConfig(std::make_unique<CSEConfigFull>());
+ CSEInfo.analyze(*MF);
+
+ CombinerHelper Helper(CSEInfo, B, /*IsPreLegalize=*/false);
+ Helper.replaceRegWith(*MRI, FromReg, ZExt.getReg(0));
+
+ EXPECT_FALSE(errorToBool(CSEInfo.verify()));
+}
+
} // namespace
>From e77b805d5461702deaebe02cb3a5bab27f9704f0 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Thu, 20 Aug 2026 14:04:03 +0530
Subject: [PATCH 2/2] [AMDGPU] Address post-legalizer CSE review feedback
Create target-constrained virtual registers instead of mutating register classes during legalization, and notify observers only for actual register replacements.
Use the standard opt-level CSE configuration directly. Remove the generic CombinerHelper bookkeeping and regression test, and leave the trap block-splitting repair for a separate change.
Co-authored-by: GPT-5 <noreply at openai.com>
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 9 --
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 140 +++++++-----------
.../AMDGPU/AMDGPUPostLegalizerCombiner.cpp | 7 +-
.../GlobalISel/global-atomic-fadd.f64.ll | 8 +-
.../CodeGen/AMDGPU/GlobalISel/global-value.ll | 73 ++++++---
llvm/unittests/CodeGen/GlobalISel/CSETest.cpp | 21 ---
6 files changed, 107 insertions(+), 151 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index e8b02f3fcce07..55c4339edc39e 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -189,13 +189,6 @@ bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
Register ToReg) const {
- // constrainRegAttrs may change attributes that are part of the CSE profile
- // for every instruction that references ToReg.
- SmallVector<MachineInstr *, 4> ToRegInstrs;
- for (MachineInstr &MI : MRI.reg_instructions(ToReg)) {
- Observer.changingInstr(MI);
- ToRegInstrs.push_back(&MI);
- }
Observer.changingAllUsesOfReg(MRI, FromReg);
if (MRI.constrainRegAttrs(ToReg, FromReg))
@@ -204,8 +197,6 @@ void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
Builder.buildCopy(FromReg, ToReg);
Observer.finishedChangingAllUsesOfReg();
- for (MachineInstr *MI : ToRegInstrs)
- Observer.changedInstr(*MI);
}
void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 6608b67363514..cacb37ed9d24c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -24,8 +24,6 @@
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
@@ -57,26 +55,6 @@ static cl::opt<bool> EnableNewLegality(
static constexpr unsigned MaxRegisterSize = 1024;
-// Register class is part of the CSE profile for every instruction that
-// references a register, so notify observers before changing it.
-static void setRegClassWithObserver(MachineIRBuilder &B,
- MachineRegisterInfo &MRI, Register Reg,
- const TargetRegisterClass *RC) {
- GISelChangeObserver *Observer = B.getMF().getObserver();
- if (!Observer) {
- MRI.setRegClass(Reg, RC);
- return;
- }
- SmallVector<MachineInstr *, 4> RegInstrs;
- for (MachineInstr &MI : MRI.reg_instructions(Reg)) {
- Observer->changingInstr(MI);
- RegInstrs.push_back(&MI);
- }
- MRI.setRegClass(Reg, RC);
- for (MachineInstr *MI : RegInstrs)
- Observer->changedInstr(*MI);
-}
-
// Round the number of elements to the next power of two elements
static LLT getPow2VectorType(LLT Ty) {
unsigned NElts = Ty.getNumElements();
@@ -3268,10 +3246,12 @@ bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
// which is a 64-bit pc-relative offset from the encoding of the $symbol
// operand to the global variable.
- LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
-
- Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
- B.getMRI()->createGenericVirtualRegister(ConstPtrTy);
+ MachineRegisterInfo &MRI = *B.getMRI();
+ LLT PCRegTy = PtrTy.getSizeInBits() == 32
+ ? LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)
+ : PtrTy;
+ Register PCReg =
+ MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PCRegTy});
if (ST.has64BitLiterals()) {
assert(GAFlags != SIInstrInfo::MO_NONE);
@@ -3290,11 +3270,10 @@ bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
MIB.addGlobalAddress(GV, Offset, GAFlags + 1);
}
- if (!B.getMRI()->getRegClassOrNull(PCReg))
- setRegClassWithObserver(B, *B.getMRI(), PCReg, &AMDGPU::SReg_64RegClass);
-
if (PtrTy.getSizeInBits() == 32)
B.buildExtract(DstReg, PCReg, 0);
+ else
+ B.buildCast(DstReg, PCReg);
return true;
}
@@ -3305,24 +3284,17 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
bool RequiresHighHalf = PtrTy.getSizeInBits() != 32;
if (RequiresHighHalf && ST.has64BitLiterals()) {
- if (!MRI.getRegClassOrNull(DstReg))
- setRegClassWithObserver(B, MRI, DstReg, &AMDGPU::SReg_64RegClass);
+ Register Addr =
+ MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PtrTy});
B.buildInstr(AMDGPU::S_MOV_B64)
- .addDef(DstReg)
+ .addDef(Addr)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
+ B.buildCast(DstReg, Addr);
return;
}
LLT I32 = LLT::integer(32);
-
- // Use the destination directly, if and only if we store the lower address
- // part only and we don't have a register class being set.
- Register AddrLo = !RequiresHighHalf && !MRI.getRegClassOrNull(DstReg)
- ? DstReg
- : MRI.createGenericVirtualRegister(I32);
-
- if (!MRI.getRegClassOrNull(AddrLo))
- setRegClassWithObserver(B, MRI, AddrLo, &AMDGPU::SReg_32RegClass);
+ Register AddrLo = MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
// Write the lower half.
B.buildInstr(AMDGPU::S_MOV_B32)
@@ -3334,31 +3306,19 @@ void AMDGPULegalizerInfo::buildAbsGlobalAddress(
assert(PtrTy.getSizeInBits() == 64 &&
"Must provide a 64-bit pointer type!");
- Register AddrHi = MRI.createGenericVirtualRegister(I32);
- MRI.setRegClass(AddrHi, &AMDGPU::SReg_32RegClass);
+ Register AddrHi =
+ MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
B.buildInstr(AMDGPU::S_MOV_B32)
.addDef(AddrHi)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_HI);
- // Use the destination directly, if and only if we don't have a register
- // class being set.
- Register AddrDst = !MRI.getRegClassOrNull(DstReg)
- ? DstReg
- : MRI.createGenericVirtualRegister(LLT::integer(64));
-
- if (!MRI.getRegClassOrNull(AddrDst))
- setRegClassWithObserver(B, MRI, AddrDst, &AMDGPU::SReg_64RegClass);
+ Register AddrDst =
+ MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, LLT::integer(64)});
B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
-
- // If we created a new register for the destination, cast the result into
- // the final output.
- if (AddrDst != DstReg)
- B.buildCast(DstReg, AddrDst);
- } else if (AddrLo != DstReg) {
- // If we created a new register for the destination, cast the result into
- // the final output.
+ B.buildCast(DstReg, AddrDst);
+ } else {
B.buildCast(DstReg, AddrLo);
}
}
@@ -7798,23 +7758,7 @@ bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
// We need a block split to make the real endpgm a terminator. We also don't
// want to break phis in successor blocks, so we can't just delete to the
// end of the block.
- // An instruction's parent block is part of its CSE profile, so notify
- // observers about the instructions moved by the split.
- GISelChangeObserver *Observer = MF->getObserver();
- SmallVector<MachineInstr *, 8> MovedInstrs;
- MachineBasicBlock::iterator SplitPoint(&MI);
- ++SplitPoint;
- if (Observer && SplitPoint != BB.end()) {
- for (MachineInstr &MovedMI : make_range(SplitPoint, BB.end())) {
- Observer->changingInstr(MovedMI);
- MovedInstrs.push_back(&MovedMI);
- }
- }
BB.splitAt(MI, false /*UpdateLiveIns*/);
- if (Observer) {
- for (MachineInstr *MovedMI : MovedInstrs)
- Observer->changedInstr(*MovedMI);
- }
MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
MF->push_back(TrapBB);
BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
@@ -8159,11 +8103,12 @@ bool AMDGPULegalizerInfo::legalizeConstHwRegRead(MachineInstr &MI,
unsigned Width) const {
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
- if (!MRI.getRegClassOrNull(DstReg))
- setRegClassWithObserver(B, MRI, DstReg, &AMDGPU::SReg_32RegClass);
+ Register Result = MRI.createVirtualRegister(
+ {&AMDGPU::SReg_32RegClass, MRI.getType(DstReg)});
B.buildInstr(AMDGPU::S_GETREG_B32_const)
- .addDef(DstReg)
+ .addDef(Result)
.addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
+ B.buildCopy(DstReg, Result);
MI.eraseFromParent();
return true;
}
@@ -8281,6 +8226,17 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
Register Def = MI.getOperand(1).getReg();
Register Use = MI.getOperand(3).getReg();
+ const TargetRegisterClass *WaveMaskRC = TRI->getWaveMaskRegClass();
+ Register NewDef =
+ MRI.createVirtualRegister({WaveMaskRC, MRI.getType(Def)});
+ Register NewUse =
+ MRI.createVirtualRegister({WaveMaskRC, MRI.getType(Use)});
+ MachineInstr *UseDef = MRI.getVRegDef(Use);
+ Helper.Observer.changingInstr(*UseDef);
+ Helper.Observer.changingAllUsesOfReg(MRI, Use);
+ MRI.replaceRegWith(Use, NewUse);
+ Helper.Observer.changedInstr(*UseDef);
+ Helper.Observer.finishedChangingAllUsesOfReg();
MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
@@ -8290,13 +8246,13 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
B.setInsertPt(B.getMBB(), BrCond->getIterator());
if (IntrID == Intrinsic::amdgcn_if) {
B.buildInstr(AMDGPU::SI_IF)
- .addDef(Def)
- .addUse(Use)
- .addMBB(UncondBrTarget);
+ .addDef(NewDef)
+ .addUse(NewUse)
+ .addMBB(UncondBrTarget);
} else {
B.buildInstr(AMDGPU::SI_ELSE)
- .addDef(Def)
- .addUse(Use)
+ .addDef(NewDef)
+ .addUse(NewUse)
.addMBB(UncondBrTarget);
}
@@ -8309,10 +8265,11 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
B.buildBr(*CondBrTarget);
}
- setRegClassWithObserver(B, MRI, Def, TRI->getWaveMaskRegClass());
- setRegClassWithObserver(B, MRI, Use, TRI->getWaveMaskRegClass());
MI.eraseFromParent();
BrCond->eraseFromParent();
+ Helper.Observer.changingAllUsesOfReg(MRI, Def);
+ MRI.replaceRegWith(Def, NewDef);
+ Helper.Observer.finishedChangingAllUsesOfReg();
return true;
}
@@ -8329,14 +8286,20 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
Register Reg = MI.getOperand(2).getReg();
+ Register NewReg = MRI.createVirtualRegister(
+ {TRI->getWaveMaskRegClass(), MRI.getType(Reg)});
+ MachineInstr *RegDef = MRI.getVRegDef(Reg);
+ Helper.Observer.changingInstr(*RegDef);
+ Helper.Observer.changingAllUsesOfReg(MRI, Reg);
+ MRI.replaceRegWith(Reg, NewReg);
+ Helper.Observer.changedInstr(*RegDef);
+ Helper.Observer.finishedChangingAllUsesOfReg();
if (Negated)
std::swap(CondBrTarget, UncondBrTarget);
B.setInsertPt(B.getMBB(), BrCond->getIterator());
- B.buildInstr(AMDGPU::SI_LOOP)
- .addUse(Reg)
- .addMBB(UncondBrTarget);
+ B.buildInstr(AMDGPU::SI_LOOP).addUse(NewReg).addMBB(UncondBrTarget);
if (Br)
Br->getOperand(0).setMBB(CondBrTarget);
@@ -8345,7 +8308,6 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MI.eraseFromParent();
BrCond->eraseFromParent();
- setRegClassWithObserver(B, MRI, Reg, TRI->getWaveMaskRegClass());
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index fbf6e73d75c23..084f67b55fd15 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -25,7 +25,6 @@
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineDominators.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Target/TargetMachine.h"
@@ -464,7 +463,6 @@ class AMDGPUPostLegalizerCombiner : public MachineFunctionPass {
} // end anonymous namespace
void AMDGPUPostLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
- AU.addRequired<TargetPassConfig>();
AU.setPreservesCFG();
getSelectionDAGFallbackAnalysisUsage(AU);
AU.addRequired<GISelValueTrackingAnalysisLegacy>();
@@ -486,7 +484,6 @@ AMDGPUPostLegalizerCombiner::AMDGPUPostLegalizerCombiner(bool IsOptNone)
bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
if (MF.getProperties().hasFailedISel())
return false;
- const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
const Function &F = MF.getFunction();
bool EnableOpt =
MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
@@ -503,7 +500,8 @@ bool AMDGPUPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
GISelCSEAnalysisWrapper &Wrapper =
getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
- GISelCSEInfo *CSEInfo = &Wrapper.get(TPC.getCSEConfig());
+ GISelCSEInfo *CSEInfo =
+ &Wrapper.get(getStandardCSEConfigForOpt(MF.getTarget().getOptLevel()));
CombinerInfo CInfo(/*AllowIllegalOps*/ false, /*ShouldLegalizeIllegal*/ true,
LI, EnableOpt, F.hasOptSize(), F.hasMinSize());
@@ -521,7 +519,6 @@ char AMDGPUPostLegalizerCombiner::ID = 0;
INITIALIZE_PASS_BEGIN(AMDGPUPostLegalizerCombiner, DEBUG_TYPE,
"Combine AMDGPU machine instrs after legalization", false,
false)
-INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
INITIALIZE_PASS_END(AMDGPUPostLegalizerCombiner, DEBUG_TYPE,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll
index 6f4067da89203..8dd420cf557d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f64.ll
@@ -20,8 +20,8 @@ define amdgpu_ps void @global_atomic_fadd_f64_no_rtn_atomicrmw(ptr addrspace(1)
; GFX90A-NEXT: bb.2.atomicrmw.start:
; GFX90A-NEXT: successors: %bb.3(0x04000000), %bb.2(0x7c000000)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI %23, %bb.2, [[S_MOV_B64_]], %bb.1
- ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %21, %bb.2
+ ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI %24, %bb.2, [[S_MOV_B64_]], %bb.1
+ ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %22, %bb.2
; GFX90A-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_ADD_F64_e64 0, [[PHI1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
; GFX90A-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[V_ADD_F64_e64_]], %subreg.sub0_sub1, [[PHI1]], %subreg.sub2_sub3
; GFX90A-NEXT: [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_CMPSWAP_X2_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic monotonic (i64) on %ir.ptr, addrspace 1)
@@ -68,8 +68,8 @@ define amdgpu_ps double @global_atomic_fadd_f64_rtn_atomicrmw(ptr addrspace(1) %
; GFX90A-NEXT: bb.2.atomicrmw.start:
; GFX90A-NEXT: successors: %bb.3(0x04000000), %bb.2(0x7c000000)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI %27, %bb.2, [[S_MOV_B64_]], %bb.1
- ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %25, %bb.2
+ ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI %28, %bb.2, [[S_MOV_B64_]], %bb.1
+ ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vreg_64_align2 = PHI [[GLOBAL_LOAD_DWORDX2_]], %bb.1, %26, %bb.2
; GFX90A-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_ADD_F64_e64 0, [[PHI1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
; GFX90A-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[V_ADD_F64_e64_]], %subreg.sub0_sub1, [[PHI1]], %subreg.sub2_sub3
; GFX90A-NEXT: [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN:%[0-9]+]]:vreg_64_align2 = GLOBAL_ATOMIC_CMPSWAP_X2_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic monotonic (i64) on %ir.ptr, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
index ee34c43485da0..664ff3fb31d2c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
@@ -18,7 +18,8 @@ define ptr addrspace(4) @external_constant_got() {
; GCN-LABEL: name: external_constant_got
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant, target-flags(amdgpu-gotprel32-hi) @external_constant, implicit-def $scc
- ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load (p4) from got, addrspace 4)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY [[SI_PC_ADD_REL_OFFSET]](p4)
+ ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p4) :: (dereferenceable invariant load (p4) from got, addrspace 4)
; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](p4)
; GCN-NEXT: $vgpr0 = COPY [[UV]](i32)
; GCN-NEXT: $vgpr1 = COPY [[UV1]](i32)
@@ -28,8 +29,11 @@ define ptr addrspace(4) @external_constant_got() {
; GCN-PAL: bb.1 (%ir-block.0):
; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @external_constant
; GCN-PAL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-hi) @external_constant
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](i32)
- ; GCN-PAL-NEXT: $vgpr1 = COPY [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[MV:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[S_MOV_B32_]](i32), [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[MV]](i64)
+ ; GCN-PAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[INTTOPTR]](p4)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; GCN-PAL-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
ret ptr addrspace(4) @external_constant
}
@@ -39,7 +43,8 @@ define ptr addrspace(1) @external_global_got() {
; GCN-LABEL: name: external_global_got
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_global, target-flags(amdgpu-gotprel32-hi) @external_global, implicit-def $scc
- ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load (p1) from got, addrspace 4)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY [[SI_PC_ADD_REL_OFFSET]](p4)
+ ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p4) :: (dereferenceable invariant load (p1) from got, addrspace 4)
; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](p1)
; GCN-NEXT: $vgpr0 = COPY [[UV]](i32)
; GCN-NEXT: $vgpr1 = COPY [[UV1]](i32)
@@ -49,8 +54,11 @@ define ptr addrspace(1) @external_global_got() {
; GCN-PAL: bb.1 (%ir-block.0):
; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @external_global
; GCN-PAL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-hi) @external_global
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](i32)
- ; GCN-PAL-NEXT: $vgpr1 = COPY [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[MV:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[S_MOV_B32_]](i32), [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[MV]](i64)
+ ; GCN-PAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[INTTOPTR]](p1)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; GCN-PAL-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
ret ptr addrspace(1) @external_global
}
@@ -60,7 +68,8 @@ define ptr addrspace(999) @external_other_got() {
; GCN-LABEL: name: external_other_got
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_other, target-flags(amdgpu-gotprel32-hi) @external_other, implicit-def $scc
- ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p999) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load (p999) from got, addrspace 4)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY [[SI_PC_ADD_REL_OFFSET]](p4)
+ ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p999) = G_LOAD [[COPY]](p4) :: (dereferenceable invariant load (p999) from got, addrspace 4)
; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](p999)
; GCN-NEXT: $vgpr0 = COPY [[UV]](i32)
; GCN-NEXT: $vgpr1 = COPY [[UV1]](i32)
@@ -70,8 +79,11 @@ define ptr addrspace(999) @external_other_got() {
; GCN-PAL: bb.1 (%ir-block.0):
; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @external_other
; GCN-PAL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-hi) @external_other
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](i32)
- ; GCN-PAL-NEXT: $vgpr1 = COPY [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[MV:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[S_MOV_B32_]](i32), [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p999) = G_INTTOPTR [[MV]](i64)
+ ; GCN-PAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[INTTOPTR]](p999)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; GCN-PAL-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
ret ptr addrspace(999) @external_other
}
@@ -81,7 +93,8 @@ define ptr addrspace(4) @internal_constant_pcrel() {
; GCN-LABEL: name: internal_constant_pcrel
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_constant, target-flags(amdgpu-rel32-hi) @internal_constant, implicit-def $scc
- ; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY [[SI_PC_ADD_REL_OFFSET]](p4)
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](p4)
; GCN-NEXT: $vgpr0 = COPY [[UV]](i32)
; GCN-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -90,8 +103,11 @@ define ptr addrspace(4) @internal_constant_pcrel() {
; GCN-PAL: bb.1 (%ir-block.0):
; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @internal_constant
; GCN-PAL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-hi) @internal_constant
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](i32)
- ; GCN-PAL-NEXT: $vgpr1 = COPY [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[MV:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[S_MOV_B32_]](i32), [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[MV]](i64)
+ ; GCN-PAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[INTTOPTR]](p4)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; GCN-PAL-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
ret ptr addrspace(4) @internal_constant
}
@@ -101,7 +117,8 @@ define ptr addrspace(1) @internal_global_pcrel() {
; GCN-LABEL: name: internal_global_pcrel
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p1) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_global, target-flags(amdgpu-rel32-hi) @internal_global, implicit-def $scc
- ; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p1)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY [[SI_PC_ADD_REL_OFFSET]](p1)
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](p1)
; GCN-NEXT: $vgpr0 = COPY [[UV]](i32)
; GCN-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -110,8 +127,11 @@ define ptr addrspace(1) @internal_global_pcrel() {
; GCN-PAL: bb.1 (%ir-block.0):
; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @internal_global
; GCN-PAL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-hi) @internal_global
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](i32)
- ; GCN-PAL-NEXT: $vgpr1 = COPY [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[MV:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[S_MOV_B32_]](i32), [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[MV]](i64)
+ ; GCN-PAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[INTTOPTR]](p1)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; GCN-PAL-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
ret ptr addrspace(1) @internal_global
}
@@ -121,7 +141,8 @@ define ptr addrspace(999) @internal_other_pcrel() {
; GCN-LABEL: name: internal_other_pcrel
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p999) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_other, target-flags(amdgpu-rel32-hi) @internal_other, implicit-def $scc
- ; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p999)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p999) = COPY [[SI_PC_ADD_REL_OFFSET]](p999)
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](p999)
; GCN-NEXT: $vgpr0 = COPY [[UV]](i32)
; GCN-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -130,8 +151,11 @@ define ptr addrspace(999) @internal_other_pcrel() {
; GCN-PAL: bb.1 (%ir-block.0):
; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @internal_other
; GCN-PAL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-hi) @internal_other
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](i32)
- ; GCN-PAL-NEXT: $vgpr1 = COPY [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[MV:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[S_MOV_B32_]](i32), [[S_MOV_B32_1]](i32)
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p999) = G_INTTOPTR [[MV]](i64)
+ ; GCN-PAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[INTTOPTR]](p999)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[UV]](i32)
+ ; GCN-PAL-NEXT: $vgpr1 = COPY [[UV1]](i32)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
ret ptr addrspace(999) @internal_other
}
@@ -141,7 +165,8 @@ define ptr addrspace(6) @external_constant32_got() {
; GCN-LABEL: name: external_constant32_got
; GCN: bb.1 (%ir-block.0):
; GCN-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant32, target-flags(amdgpu-gotprel32-hi) @external_constant32, implicit-def $scc
- ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load (p4) from got, addrspace 4)
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY [[SI_PC_ADD_REL_OFFSET]](p4)
+ ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p4) :: (dereferenceable invariant load (p4) from got, addrspace 4)
; GCN-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[LOAD]](p4)
; GCN-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[PTRTOINT]](s64)
; GCN-NEXT: [[INTTOPTR:%[0-9]+]]:_(p6) = G_INTTOPTR [[TRUNC]](i32)
@@ -150,8 +175,9 @@ define ptr addrspace(6) @external_constant32_got() {
;
; GCN-PAL-LABEL: name: external_constant32_got
; GCN-PAL: bb.1 (%ir-block.0):
- ; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(p6) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @external_constant32
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](p6)
+ ; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @external_constant32
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p6) = G_INTTOPTR [[S_MOV_B32_]](i32)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[INTTOPTR]](p6)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0
ret ptr addrspace(6) @external_constant32
}
@@ -169,8 +195,9 @@ define ptr addrspace(6) @internal_constant32_pcrel() {
;
; GCN-PAL-LABEL: name: internal_constant32_pcrel
; GCN-PAL: bb.1 (%ir-block.0):
- ; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(p6) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @internal_constant32
- ; GCN-PAL-NEXT: $vgpr0 = COPY [[S_MOV_B32_]](p6)
+ ; GCN-PAL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 target-flags(amdgpu-abs32-lo) @internal_constant32
+ ; GCN-PAL-NEXT: [[INTTOPTR:%[0-9]+]]:_(p6) = G_INTTOPTR [[S_MOV_B32_]](i32)
+ ; GCN-PAL-NEXT: $vgpr0 = COPY [[INTTOPTR]](p6)
; GCN-PAL-NEXT: SI_RETURN implicit $vgpr0
ret ptr addrspace(6) @internal_constant32
}
diff --git a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
index 0211a230f5efb..6255ac3293a08 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
@@ -9,7 +9,6 @@
#include "GISelMITest.h"
#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
-#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "gtest/gtest.h"
namespace {
@@ -669,24 +668,4 @@ TEST_F(AArch64GISelMITest, TestConstantFoldICMP) {
EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}
-TEST_F(AMDGPUGISelMITest, TestReplaceRegWithUpdatesCSE) {
- setUp();
- if (!TM)
- GTEST_SKIP();
-
- LLT S64 = LLT::scalar(64);
- auto ZExt = B.buildZExt(S64, Copies[0]);
- Register FromReg = MRI->createGenericVirtualRegister(S64);
- MRI->setRegClass(FromReg, MRI->getTargetRegisterInfo()->getPointerRegClass());
-
- GISelCSEInfo CSEInfo;
- CSEInfo.setCSEConfig(std::make_unique<CSEConfigFull>());
- CSEInfo.analyze(*MF);
-
- CombinerHelper Helper(CSEInfo, B, /*IsPreLegalize=*/false);
- Helper.replaceRegWith(*MRI, FromReg, ZExt.getReg(0));
-
- EXPECT_FALSE(errorToBool(CSEInfo.verify()));
-}
-
} // namespace
More information about the llvm-commits
mailing list