[llvm-branch-commits] [llvm] [RegAlloc] Add enableCSRFirstTimeMinCost to let RegAlloc has more choice (PR #239530)
Piyou Chen via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 12 22:32:40 PST 2026
https://github.com/BeMg updated https://github.com/llvm/llvm-project/pull/239530
>From 06b865788bc7558e44ccffd71f66d4b2174b03fa Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Thu, 5 Nov 2026 23:18:29 -0800
Subject: [PATCH 1/3] [RegAlloc] Make tryAssignCSRFirstTime consider
Remat/Spill
---
.../include/llvm/CodeGen/TargetRegisterInfo.h | 8 +
llvm/lib/CodeGen/RegAllocGreedy.cpp | 180 +++++++++++++++++-
llvm/lib/CodeGen/RegAllocGreedy.h | 15 ++
3 files changed, 201 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index fc3b4d6dda844..3888f141b5370 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -899,6 +899,14 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
return 30;
}
+ /// Return true if the greedy register allocator should take
+ /// the lowest-cost choice before using an unused CSR.
+ /// This uses tryAssignCSRFirstTimeMinCost instead of
+ /// tryAssignCSRFirstTime.
+ virtual bool enableCSRFirstTimeMinCost(const MachineFunction &MF) const {
+ return false;
+ }
+
/// Returns true if the target requires (and can make use of) the register
/// scavenger.
virtual bool requiresRegisterScavenging(const MachineFunction &MF) const {
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index f7655e97b9942..acce796d5ff65 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2471,6 +2471,49 @@ BlockFrequency RAGreedy::calcRematCost(const LiveInterval &VirtReg,
return HasUse ? RematCost : BlockFrequency::max();
}
+/// Return true if the spiller can rematerialize each use of \p LI.
+/// And it can rematerialize for only one def.
+static bool isRematerializableFromOriginal(const LiveInterval &LI,
+ const LiveIntervals &LIS,
+ const VirtRegMap &VRM,
+ const MachineRegisterInfo &MRI,
+ const TargetInstrInfo &TII) {
+ const LiveInterval &OrigLI = LIS.getInterval(VRM.getOriginal(LI.reg()));
+ const MachineInstr *Def = nullptr;
+ for (const VNInfo *VNI : OrigLI.valnos) {
+ if (VNI->isUnused())
+ continue;
+ // Maybe more than one value def for VirtReg
+ if (VNI->isPHIDef() || Def)
+ return false;
+ Def = LIS.getInstructionFromIndex(VNI->def);
+ if (!Def || !TII.isReMaterializable(*Def))
+ return false;
+ }
+ if (!Def)
+ return false;
+ for (const MachineInstr &MI : MRI.use_nodbg_instructions(LI.reg()))
+ if (!VirtRegAuxInfo::allUsesAvailableAt(Def, LIS.getInstructionIndex(MI),
+ LIS, MRI, TII))
+ return false;
+ return true;
+}
+
+/// The cost of rematerializing \p LI before each of its uses.
+static BlockFrequency calcRematCostAtUses(const LiveInterval &LI,
+ const MachineRegisterInfo &MRI,
+ const SpillPlacement &SpillPlacer) {
+ uint64_t RematCost = 0;
+ SmallPtrSet<const MachineInstr *, 8> Seen;
+ for (const MachineInstr &MI : MRI.use_nodbg_instructions(LI.reg())) {
+ if (MI.isMetaInstruction() || !Seen.insert(&MI).second)
+ continue;
+ RematCost += SpillPlacer.getBlockFrequency(MI.getParent()->getNumber())
+ .getFrequency();
+ }
+ return BlockFrequency(RematCost);
+}
+
/// Using a CSR for the first time has a cost because it causes push|pop
/// to be added to prologue|epilogue. Splitting a cold section of the live
/// range can have lower cost than using the CSR for the first time;
@@ -2524,6 +2567,135 @@ MCRegister RAGreedy::tryAssignCSRFirstTime(
return PhysReg;
}
+// Like tryAssignCSRFirstTime before assign the unused CSR to VirtReg, try
+// other choices. There are four results:
+// 1. Pre-split VirtReg
+// 2. Remat/Spill VirtReg
+// 3. Evict rematerializable live ranges if that reg is not unused CSR.
+// 4. Use the CSR for this VirtReg
+// The goal is using the less cost register.
+MCRegister RAGreedy::tryAssignCSRFirstTimeMinCost(
+ const LiveInterval &VirtReg, AllocationOrder &Order, MCRegister PhysReg,
+ uint8_t &CostPerUseLimit, SmallVectorImpl<Register> &NewVRegs) {
+ // Compute Remat cost.
+ std::optional<BlockFrequency> RematCost;
+ if (VirtReg.isSpillable() &&
+ isRematerializableFromOriginal(VirtReg, *LIS, *VRM, *MRI, *TII))
+ RematCost = calcRematCostAtUses(VirtReg, *MRI, *SpillPlacer);
+
+ std::optional<BlockFrequency> SpillRematCost;
+ if (VirtReg.isSpillable()) {
+ SpillRematCost = calcSpillCost(VirtReg);
+ if (RematCost)
+ SpillRematCost = std::min(*SpillRematCost, *RematCost);
+ }
+
+ LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
+
+ if (Stage == RS_Spill && VirtReg.isSpillable()) {
+ // We choose spill or rematerialize over using the CSR for the first time if
+ // the spill cost is lower than CSRCost.
+ if (*SpillRematCost >= CSRCost)
+ return PhysReg;
+
+ // We are going to spill or rematerialize, set CostPerUseLimit to 1 to make
+ // sure that we will not use a callee-saved register in tryEvict.
+ CostPerUseLimit = 1;
+ return MCRegister();
+ }
+
+ if (Stage < RS_Split) {
+ SA->analyze(&VirtReg);
+ // A pre-split must cost less than CSRCost.
+ // CSRCost is the cost for place the CSR save/restore. And it be set by
+ // target.
+ BlockFrequency SplitCost = CSRCost;
+ unsigned NumCands = 0;
+ unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, SplitCost,
+ NumCands, true /*IgnoreCSR*/);
+
+ bool SpillRematCostsLess = SpillRematCost && *SpillRematCost < SplitCost;
+ if (BestCand != NoCand && !SpillRematCostsLess) {
+ doRegionSplit(VirtReg, BestCand, false /*HasCompact*/, NewVRegs);
+ return MCRegister();
+ }
+ }
+
+ bool SpillCostsLess = SpillRematCost && *SpillRematCost < CSRCost;
+ BlockFrequency EvictLimit = SpillCostsLess ? *SpillRematCost : CSRCost;
+ // Use Evict Reg
+ if (MCRegister Reg =
+ tryEvictRematForCSR(VirtReg, Order, NewVRegs, EvictLimit))
+ return Reg;
+
+ // Use CSR
+ if (!SpillCostsLess)
+ return PhysReg;
+
+ // Use Spill/Remat
+ // We are going to spill, set CostPerUseLimit to 1 to make sure that
+ // we will not use a callee-saved register in tryEvict.
+ ExtraInfo->setStage(VirtReg, RS_Spill);
+ CostPerUseLimit = 1;
+ return MCRegister();
+}
+
+/// Return the cost of evicting and rematerializing all the live ranges that
+/// interfere with \p VirtReg in \p PhysReg.
+/// Return std::nullopt when
+/// 1. has lower Cascade than \p Cascade.
+/// 2. can't be spilled.
+/// 3. can't be rematerialized.
+std::optional<BlockFrequency>
+RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
+ unsigned Cascade) const {
+ BlockFrequency Cost(0);
+ SmallPtrSet<const LiveInterval *, 4> Seen;
+ for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
+ for (const LiveInterval *Intf :
+ Matrix->query(VirtReg, Unit).interferingVRegs()) {
+ if (!Seen.insert(Intf).second)
+ continue;
+ if (ExtraInfo->getCascade(Intf->reg()) >= Cascade ||
+ !Intf->isSpillable() ||
+ !isRematerializableFromOriginal(*Intf, *LIS, *VRM, *MRI, *TII))
+ return std::nullopt;
+ Cost += calcRematCostAtUses(*Intf, *MRI, *SpillPlacer);
+ }
+ }
+ return Cost;
+}
+
+/// Before \p VirtReg uses a CSR for the first time, look for a register that
+/// is not an unused CSR and that only rematerializable live ranges occupy.
+/// If rematerializing them costs less than \p MaxCost, evict them and
+/// return the register.
+MCRegister RAGreedy::tryEvictRematForCSR(const LiveInterval &VirtReg,
+ AllocationOrder &Order,
+ SmallVectorImpl<Register> &NewVRegs,
+ BlockFrequency MaxCost) {
+ unsigned Cascade = ExtraInfo->getCascadeOrCurrentNext(VirtReg.reg());
+ MCRegister BestReg;
+ BlockFrequency BestCost = MaxCost;
+ for (MCRegister PhysReg : Order.getOrder()) {
+ if (EvictAdvisor->getUnusedCalleeSavedReg(VirtReg, PhysReg) ||
+ Matrix->checkInterference(VirtReg, PhysReg) !=
+ LiveRegMatrix::IK_VirtReg)
+ continue;
+ std::optional<BlockFrequency> Cost =
+ calcEvictRematCost(VirtReg, PhysReg, Cascade);
+ if (Cost && *Cost < BestCost) {
+ BestCost = *Cost;
+ BestReg = PhysReg;
+ }
+ }
+ if (!BestReg)
+ return MCRegister();
+
+ evictInterference(VirtReg, BestReg, NewVRegs);
+ return BestReg;
+}
+
void RAGreedy::aboutToRemoveInterval(const LiveInterval &LI) {
// Do not keep invalid information around.
SetOfBrokenHints.remove(&LI);
@@ -2791,8 +2963,12 @@ MCRegister RAGreedy::selectOrSplitImpl(const LiveInterval &VirtReg,
if (CSRCost.getFrequency() &&
EvictAdvisor->getUnusedCalleeSavedReg(VirtReg, PhysReg) &&
NewVRegs.empty()) {
- MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
- CostPerUseLimit, NewVRegs);
+ MCRegister CSRReg =
+ TRI->enableCSRFirstTimeMinCost(*MF)
+ ? tryAssignCSRFirstTimeMinCost(VirtReg, Order, PhysReg,
+ CostPerUseLimit, NewVRegs)
+ : tryAssignCSRFirstTime(VirtReg, Order, PhysReg, CostPerUseLimit,
+ NewVRegs);
if (CSRReg || !NewVRegs.empty())
// Return now if we decide to use a CSR or create new vregs due to
// pre-splitting.
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 138da362a1e76..c9c1d68f623c0 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -365,6 +365,21 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
/// rematerialization is not possible.
BlockFrequency calcRematCost(const LiveInterval &VirtReg,
AllocationOrder &Order) const;
+ /// Like tryAssignCSRFirstTime, but take the choice with the lowest cost of:
+ /// a pre-split, spilling or rematerializing VirtReg, evicting
+ /// rematerializable live ranges, and the first use of the CSR.
+ MCRegister tryAssignCSRFirstTimeMinCost(const LiveInterval &VirtReg,
+ AllocationOrder &Order,
+ MCRegister PhysReg,
+ uint8_t &CostPerUseLimit,
+ SmallVectorImpl<Register> &NewVRegs);
+ std::optional<BlockFrequency>
+ calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
+ unsigned Cascade) const;
+ MCRegister tryEvictRematForCSR(const LiveInterval &VirtReg,
+ AllocationOrder &Order,
+ SmallVectorImpl<Register> &NewVRegs,
+ BlockFrequency MaxCost);
BlockFrequency calcSpillCost(const LiveInterval &LI);
void initializeCSRCost();
MCRegister tryBlockSplit(const LiveInterval &, AllocationOrder &,
>From 969a76f208926513080756d16ef646d7cc405537 Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Sun, 8 Nov 2026 18:07:25 -0800
Subject: [PATCH 2/3] !fixup fix format
---
llvm/lib/CodeGen/RegAllocGreedy.h | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index c9c1d68f623c0..4335343942320 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -373,9 +373,9 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
MCRegister PhysReg,
uint8_t &CostPerUseLimit,
SmallVectorImpl<Register> &NewVRegs);
- std::optional<BlockFrequency>
- calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
- unsigned Cascade) const;
+ std::optional<BlockFrequency> calcEvictRematCost(const LiveInterval &VirtReg,
+ MCRegister PhysReg,
+ unsigned Cascade) const;
MCRegister tryEvictRematForCSR(const LiveInterval &VirtReg,
AllocationOrder &Order,
SmallVectorImpl<Register> &NewVRegs,
>From 9babcf5843331848466a488f0840b8778dd7ce00 Mon Sep 17 00:00:00 2001
From: BeMg <gccbg04538 at gmail.com>
Date: Fri, 13 Nov 2026 14:07:07 +0800
Subject: [PATCH 3/3] !fixup Merge the tryAssignCSRFirstTimeMinCost into
tryAssignCSRFirstTime and make it enable by default
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Note: I still added a hook, shouldSpillInsteadOfFirstCSRUse, to avoid a test case crash. The crash happens because AMDGPU has a register class that isn’t suitable for this kind of spill.
I keep it enable by default but disable for AMDGPU.
---
.../include/llvm/CodeGen/TargetRegisterInfo.h | 14 +-
llvm/lib/CodeGen/RegAllocGreedy.cpp | 161 +++----
llvm/lib/CodeGen/RegAllocGreedy.h | 13 +-
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 9 +
llvm/test/CodeGen/AArch64/call-rv-marker.ll | 51 +-
.../test/CodeGen/AArch64/sme-za-exceptions.ll | 47 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 306 ++++++------
llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll | 396 +++++++---------
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 199 ++++----
...nfloop-subrange-spill-inspect-subrange.mir | 40 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 37 +-
llvm/test/CodeGen/AMDGPU/issue48473.mir | 2 +-
.../AMDGPU/large-avgpr-assign-last.mir | 10 +-
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll | 404 ++++++++--------
llvm/test/CodeGen/AMDGPU/maximumnum.ll | 447 +++++++++---------
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll | 404 ++++++++--------
llvm/test/CodeGen/AMDGPU/minimumnum.ll | 447 +++++++++---------
llvm/test/CodeGen/AMDGPU/select.f16.ll | 168 +++----
llvm/test/CodeGen/RISCV/csr-first-use-cost.ll | 18 +-
.../RISCV/exception-pointer-register.ll | 36 +-
.../X86/2007-08-09-IllegalX86-64Asm.ll | 24 +-
.../CodeGen/X86/2007-11-30-LoadFolding-Bug.ll | 73 ++-
llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll | 35 +-
.../CodeGen/X86/2008-04-17-CoalescerBug.ll | 30 +-
.../CodeGen/X86/apx/memfold-origVNI-crash.ll | 90 ++--
llvm/test/CodeGen/X86/atom-fixup-lea2.ll | 4 +-
llvm/test/CodeGen/X86/call-rv-marker.ll | 9 +-
llvm/test/CodeGen/X86/csr-split.ll | 9 +-
.../CodeGen/X86/expand-large-fp-optnone.ll | 110 ++---
llvm/test/CodeGen/X86/i386-shrink-wrapping.ll | 4 +-
llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll | 28 +-
llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll | 40 +-
.../CodeGen/X86/speculative-load-hardening.ll | 36 +-
llvm/test/CodeGen/X86/throws-cfi-fp.ll | 7 +-
llvm/test/CodeGen/X86/throws-cfi-no-fp.ll | 2 +-
35 files changed, 1756 insertions(+), 1954 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 3888f141b5370..880a48df56466 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -899,12 +899,14 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
return 30;
}
- /// Return true if the greedy register allocator should take
- /// the lowest-cost choice before using an unused CSR.
- /// This uses tryAssignCSRFirstTimeMinCost instead of
- /// tryAssignCSRFirstTime.
- virtual bool enableCSRFirstTimeMinCost(const MachineFunction &MF) const {
- return false;
+ /// Return true if the RegAlloc can spill instead of using an unused CSR,
+ /// when the spill costs less than the CSR cost. Return false if a spill
+ /// on this target is not comparable with the CSR cost, for example when
+ /// a spill does not go to the stack.
+ virtual bool
+ shouldSpillInsteadOfFirstCSRUse(const MachineFunction &MF,
+ const LiveInterval &VirtReg) const {
+ return true;
}
/// Returns true if the target requires (and can make use of) the register
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index acce796d5ff65..55c11987c0898 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2471,8 +2471,8 @@ BlockFrequency RAGreedy::calcRematCost(const LiveInterval &VirtReg,
return HasUse ? RematCost : BlockFrequency::max();
}
-/// Return true if the spiller can rematerialize each use of \p LI.
-/// And it can rematerialize for only one def.
+/// Return true if the spiller can rematerialize each use of LI.
+/// And it can rematerialize for only one def, which does not load.
static bool isRematerializableFromOriginal(const LiveInterval &LI,
const LiveIntervals &LIS,
const VirtRegMap &VRM,
@@ -2483,17 +2483,18 @@ static bool isRematerializableFromOriginal(const LiveInterval &LI,
for (const VNInfo *VNI : OrigLI.valnos) {
if (VNI->isUnused())
continue;
- // Maybe more than one value def for VirtReg
+ // The original live range must have only one def.
if (VNI->isPHIDef() || Def)
return false;
Def = LIS.getInstructionFromIndex(VNI->def);
- if (!Def || !TII.isReMaterializable(*Def))
+ if (!Def || !TII.isReMaterializable(*Def) || Def->mayLoad())
return false;
}
if (!Def)
return false;
for (const MachineInstr &MI : MRI.use_nodbg_instructions(LI.reg()))
- if (!VirtRegAuxInfo::allUsesAvailableAt(Def, LIS.getInstructionIndex(MI),
+ if (!canGuaranteeAssignmentAfterRemat(LI.reg(), MI) ||
+ !VirtRegAuxInfo::allUsesAvailableAt(Def, LIS.getInstructionIndex(MI),
LIS, MRI, TII))
return false;
return true;
@@ -2525,6 +2526,10 @@ MCRegister RAGreedy::tryAssignCSRFirstTime(
uint8_t &CostPerUseLimit, SmallVectorImpl<Register> &NewVRegs) {
LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
+ // For RS_Done, just use the CSR.
+ if (Stage > RS_Spill)
+ return PhysReg;
+
if (Stage == RS_Spill && VirtReg.isSpillable()) {
// We choose spill or rematerialize over using the CSR for the first time if
// the spill cost is lower than CSRCost.
@@ -2539,116 +2544,61 @@ MCRegister RAGreedy::tryAssignCSRFirstTime(
CostPerUseLimit = 1;
return MCRegister();
}
+
+ // The lowest cost of rematerializing or spilling VirtReg.
+ BlockFrequency SpillCost = BlockFrequency::max();
+ if (VirtReg.isSpillable()) {
+ if (isRematerializableFromOriginal(VirtReg, *LIS, *VRM, *MRI, *TII))
+ SpillCost = calcRematCostAtUses(VirtReg, *MRI, *SpillPlacer);
+ if (TRI->shouldSpillInsteadOfFirstCSRUse(*MF, VirtReg))
+ SpillCost = std::min(SpillCost, calcSpillCost(VirtReg));
+ }
+
if (Stage < RS_Split) {
// We choose pre-splitting over using the CSR for the first time if
- // the cost of splitting is lower than CSRCost.
+ // the cost of splitting is lower than CSRCost and the remat/spill cost.
SA->analyze(&VirtReg);
unsigned NumCands = 0;
BlockFrequency BestCost = CSRCost; // Don't modify CSRCost.
unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, BestCost,
NumCands, true /*IgnoreCSR*/);
- if (BestCand == NoCand) {
- if (calcRematCost(VirtReg, Order) < CSRCost) {
- CostPerUseLimit = 1;
- return MCRegister();
- }
- // Use the CSR if we can't find a region split below CSRCost.
- return PhysReg;
+ if (BestCand != NoCand && BestCost <= SpillCost) {
+ // Perform the actual pre-splitting.
+ doRegionSplit(VirtReg, BestCand, false /*HasCompact*/, NewVRegs);
+ return MCRegister();
}
-
- // Perform the actual pre-splitting.
- doRegionSplit(VirtReg, BestCand, false/*HasCompact*/, NewVRegs);
- return MCRegister();
}
if (calcRematCost(VirtReg, Order) < CSRCost) {
CostPerUseLimit = 1;
return MCRegister();
}
- return PhysReg;
-}
-
-// Like tryAssignCSRFirstTime before assign the unused CSR to VirtReg, try
-// other choices. There are four results:
-// 1. Pre-split VirtReg
-// 2. Remat/Spill VirtReg
-// 3. Evict rematerializable live ranges if that reg is not unused CSR.
-// 4. Use the CSR for this VirtReg
-// The goal is using the less cost register.
-MCRegister RAGreedy::tryAssignCSRFirstTimeMinCost(
- const LiveInterval &VirtReg, AllocationOrder &Order, MCRegister PhysReg,
- uint8_t &CostPerUseLimit, SmallVectorImpl<Register> &NewVRegs) {
- // Compute Remat cost.
- std::optional<BlockFrequency> RematCost;
- if (VirtReg.isSpillable() &&
- isRematerializableFromOriginal(VirtReg, *LIS, *VRM, *MRI, *TII))
- RematCost = calcRematCostAtUses(VirtReg, *MRI, *SpillPlacer);
-
- std::optional<BlockFrequency> SpillRematCost;
- if (VirtReg.isSpillable()) {
- SpillRematCost = calcSpillCost(VirtReg);
- if (RematCost)
- SpillRematCost = std::min(*SpillRematCost, *RematCost);
- }
-
- LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
-
- if (Stage == RS_Spill && VirtReg.isSpillable()) {
- // We choose spill or rematerialize over using the CSR for the first time if
- // the spill cost is lower than CSRCost.
- if (*SpillRematCost >= CSRCost)
- return PhysReg;
-
- // We are going to spill or rematerialize, set CostPerUseLimit to 1 to make
- // sure that we will not use a callee-saved register in tryEvict.
+ // Evict rematerializable live ranges from a register that is not an unused
+ // CSR, if that costs less than CSRCost and the spill cost.
+ if (MCRegister Reg = tryEvictRematForCSR(VirtReg, Order, NewVRegs,
+ std::min(CSRCost, SpillCost)))
+ return Reg;
+ // We choose spilling or rematerialization over using the CSR for the first
+ // time if it costs less than CSRCost.
+ if (SpillCost < CSRCost) {
+ // Set CostPerUseLimit to 1 to make sure that we will not use a
+ // callee-saved register in tryEvict.
+ ExtraInfo->setStage(VirtReg, RS_Spill);
CostPerUseLimit = 1;
return MCRegister();
}
-
- if (Stage < RS_Split) {
- SA->analyze(&VirtReg);
- // A pre-split must cost less than CSRCost.
- // CSRCost is the cost for place the CSR save/restore. And it be set by
- // target.
- BlockFrequency SplitCost = CSRCost;
- unsigned NumCands = 0;
- unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, SplitCost,
- NumCands, true /*IgnoreCSR*/);
-
- bool SpillRematCostsLess = SpillRematCost && *SpillRematCost < SplitCost;
- if (BestCand != NoCand && !SpillRematCostsLess) {
- doRegionSplit(VirtReg, BestCand, false /*HasCompact*/, NewVRegs);
- return MCRegister();
- }
- }
-
- bool SpillCostsLess = SpillRematCost && *SpillRematCost < CSRCost;
- BlockFrequency EvictLimit = SpillCostsLess ? *SpillRematCost : CSRCost;
- // Use Evict Reg
- if (MCRegister Reg =
- tryEvictRematForCSR(VirtReg, Order, NewVRegs, EvictLimit))
- return Reg;
-
- // Use CSR
- if (!SpillCostsLess)
- return PhysReg;
-
- // Use Spill/Remat
- // We are going to spill, set CostPerUseLimit to 1 to make sure that
- // we will not use a callee-saved register in tryEvict.
- ExtraInfo->setStage(VirtReg, RS_Spill);
- CostPerUseLimit = 1;
- return MCRegister();
+ // Use the CSR if no other choice costs less than CSRCost.
+ return PhysReg;
}
/// Return the cost of evicting and rematerializing all the live ranges that
-/// interfere with \p VirtReg in \p PhysReg.
-/// Return std::nullopt when
-/// 1. has lower Cascade than \p Cascade.
-/// 2. can't be spilled.
+/// interfere with VirtReg in PhysReg.
+/// Return BlockFrequency::max() when one of these live ranges
+/// 1. has a Cascade that is not lower than Cascade,
+/// 2. can't be spilled, or
/// 3. can't be rematerialized.
-std::optional<BlockFrequency>
-RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
- unsigned Cascade) const {
+BlockFrequency RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg,
+ MCRegister PhysReg,
+ unsigned Cascade) const {
BlockFrequency Cost(0);
SmallPtrSet<const LiveInterval *, 4> Seen;
for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
@@ -2659,16 +2609,16 @@ RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
if (ExtraInfo->getCascade(Intf->reg()) >= Cascade ||
!Intf->isSpillable() ||
!isRematerializableFromOriginal(*Intf, *LIS, *VRM, *MRI, *TII))
- return std::nullopt;
+ return BlockFrequency::max();
Cost += calcRematCostAtUses(*Intf, *MRI, *SpillPlacer);
}
}
return Cost;
}
-/// Before \p VirtReg uses a CSR for the first time, look for a register that
+/// Before VirtReg uses a CSR for the first time, look for a register that
/// is not an unused CSR and that only rematerializable live ranges occupy.
-/// If rematerializing them costs less than \p MaxCost, evict them and
+/// If rematerializing them costs less than MaxCost, evict them and
/// return the register.
MCRegister RAGreedy::tryEvictRematForCSR(const LiveInterval &VirtReg,
AllocationOrder &Order,
@@ -2682,10 +2632,9 @@ MCRegister RAGreedy::tryEvictRematForCSR(const LiveInterval &VirtReg,
Matrix->checkInterference(VirtReg, PhysReg) !=
LiveRegMatrix::IK_VirtReg)
continue;
- std::optional<BlockFrequency> Cost =
- calcEvictRematCost(VirtReg, PhysReg, Cascade);
- if (Cost && *Cost < BestCost) {
- BestCost = *Cost;
+ BlockFrequency Cost = calcEvictRematCost(VirtReg, PhysReg, Cascade);
+ if (Cost < BestCost) {
+ BestCost = Cost;
BestReg = PhysReg;
}
}
@@ -2963,12 +2912,8 @@ MCRegister RAGreedy::selectOrSplitImpl(const LiveInterval &VirtReg,
if (CSRCost.getFrequency() &&
EvictAdvisor->getUnusedCalleeSavedReg(VirtReg, PhysReg) &&
NewVRegs.empty()) {
- MCRegister CSRReg =
- TRI->enableCSRFirstTimeMinCost(*MF)
- ? tryAssignCSRFirstTimeMinCost(VirtReg, Order, PhysReg,
- CostPerUseLimit, NewVRegs)
- : tryAssignCSRFirstTime(VirtReg, Order, PhysReg, CostPerUseLimit,
- NewVRegs);
+ MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
+ CostPerUseLimit, NewVRegs);
if (CSRReg || !NewVRegs.empty())
// Return now if we decide to use a CSR or create new vregs due to
// pre-splitting.
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 4335343942320..16273854eaaf6 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -365,17 +365,8 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
/// rematerialization is not possible.
BlockFrequency calcRematCost(const LiveInterval &VirtReg,
AllocationOrder &Order) const;
- /// Like tryAssignCSRFirstTime, but take the choice with the lowest cost of:
- /// a pre-split, spilling or rematerializing VirtReg, evicting
- /// rematerializable live ranges, and the first use of the CSR.
- MCRegister tryAssignCSRFirstTimeMinCost(const LiveInterval &VirtReg,
- AllocationOrder &Order,
- MCRegister PhysReg,
- uint8_t &CostPerUseLimit,
- SmallVectorImpl<Register> &NewVRegs);
- std::optional<BlockFrequency> calcEvictRematCost(const LiveInterval &VirtReg,
- MCRegister PhysReg,
- unsigned Cascade) const;
+ BlockFrequency calcEvictRematCost(const LiveInterval &VirtReg,
+ MCRegister PhysReg, unsigned Cascade) const;
MCRegister tryEvictRematForCSR(const LiveInterval &VirtReg,
AllocationOrder &Order,
SmallVectorImpl<Register> &NewVRegs,
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 4cacba0ed2592..a1362f34848e2 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -128,6 +128,15 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
return 27;
}
+ bool
+ shouldSpillInsteadOfFirstCSRUse(const MachineFunction &MF,
+ const LiveInterval &VirtReg) const override {
+ // An SGPR spill goes to a VGPR lane, like a CSR SGPR save, but at each use,
+ // and it needs one more WWM VGPR. getCSRFirstUseCost also models occupancy,
+ // so it is not comparable with the spill cost.
+ return false;
+ }
+
bool shouldCompactSlotIndexes(const MachineFunction &MF) const override {
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/call-rv-marker.ll b/llvm/test/CodeGen/AArch64/call-rv-marker.ll
index a2a9138f57f9a..682f0435ab0dd 100644
--- a/llvm/test/CodeGen/AArch64/call-rv-marker.ll
+++ b/llvm/test/CodeGen/AArch64/call-rv-marker.ll
@@ -89,9 +89,10 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
; CHECK-SD-NEXT: .cfi_personality 155, ___gxx_personality_v0
; CHECK-SD-NEXT: .cfi_lsda 16, Lexception0
; CHECK-SD-NEXT: ; %bb.0: ; %entry
-; CHECK-SD-NEXT: stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
-; CHECK-SD-NEXT: stp x29, x30, [sp, #16] ; 16-byte Folded Spill
-; CHECK-SD-NEXT: .cfi_def_cfa_offset 32
+; CHECK-SD-NEXT: sub sp, sp, #48
+; CHECK-SD-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill
+; CHECK-SD-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill
+; CHECK-SD-NEXT: .cfi_def_cfa_offset 48
; CHECK-SD-NEXT: .cfi_offset w30, -8
; CHECK-SD-NEXT: .cfi_offset w29, -16
; CHECK-SD-NEXT: .cfi_offset w19, -24
@@ -104,16 +105,17 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
; CHECK-SD-NEXT: bl _objc_object
; CHECK-SD-NEXT: Ltmp1: ; EH_LABEL
; CHECK-SD-NEXT: ; %bb.1: ; %invoke.cont
-; CHECK-SD-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
; CHECK-SD-NEXT: mov x0, x19
-; CHECK-SD-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-SD-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload
+; CHECK-SD-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload
+; CHECK-SD-NEXT: add sp, sp, #48
; CHECK-SD-NEXT: b _objc_release
; CHECK-SD-NEXT: LBB3_2: ; %lpad
; CHECK-SD-NEXT: Ltmp2: ; EH_LABEL
-; CHECK-SD-NEXT: mov x20, x0
+; CHECK-SD-NEXT: str x0, [sp, #8] ; 8-byte Spill
; CHECK-SD-NEXT: mov x0, x19
; CHECK-SD-NEXT: bl _objc_release
-; CHECK-SD-NEXT: mov x0, x20
+; CHECK-SD-NEXT: ldr x0, [sp, #8] ; 8-byte Reload
; CHECK-SD-NEXT: bl __Unwind_Resume
; CHECK-SD-NEXT: Lfunc_end0:
; CHECK-SD-NEXT: .cfi_endproc
@@ -147,9 +149,10 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
; CHECK-GI-NEXT: .cfi_personality 155, ___gxx_personality_v0
; CHECK-GI-NEXT: .cfi_lsda 16, Lexception0
; CHECK-GI-NEXT: ; %bb.0: ; %entry
-; CHECK-GI-NEXT: stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x29, x30, [sp, #16] ; 16-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 32
+; CHECK-GI-NEXT: sub sp, sp, #48
+; CHECK-GI-NEXT: stp x20, x19, [sp, #16] ; 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x29, x30, [sp, #32] ; 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 48
; CHECK-GI-NEXT: .cfi_offset w30, -8
; CHECK-GI-NEXT: .cfi_offset w29, -16
; CHECK-GI-NEXT: .cfi_offset w19, -24
@@ -167,19 +170,20 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
; CHECK-GI-NEXT: mov x0, x19
; CHECK-GI-NEXT: Lloh1:
; CHECK-GI-NEXT: ldr x1, [x1, _objc_release at GOTPAGEOFF]
-; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
-; CHECK-GI-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp x20, x19, [sp, #16] ; 16-byte Folded Reload
+; CHECK-GI-NEXT: add sp, sp, #48
; CHECK-GI-NEXT: br x1
; CHECK-GI-NEXT: LBB3_2: ; %lpad
; CHECK-GI-NEXT: Ltmp2: ; EH_LABEL
; CHECK-GI-NEXT: Lloh2:
; CHECK-GI-NEXT: adrp x8, _objc_release at GOTPAGE
-; CHECK-GI-NEXT: mov x20, x0
-; CHECK-GI-NEXT: mov x0, x19
; CHECK-GI-NEXT: Lloh3:
; CHECK-GI-NEXT: ldr x8, [x8, _objc_release at GOTPAGEOFF]
+; CHECK-GI-NEXT: str x0, [sp, #8] ; 8-byte Spill
+; CHECK-GI-NEXT: mov x0, x19
; CHECK-GI-NEXT: blr x8
-; CHECK-GI-NEXT: mov x0, x20
+; CHECK-GI-NEXT: ldr x0, [sp, #8] ; 8-byte Reload
; CHECK-GI-NEXT: bl __Unwind_Resume
; CHECK-GI-NEXT: .loh AdrpLdrGot Lloh0, Lloh1
; CHECK-GI-NEXT: .loh AdrpLdrGot Lloh2, Lloh3
@@ -260,18 +264,17 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 {
; CHECK-SD-NEXT: ret
; CHECK-SD-NEXT: LBB4_3: ; %lpad1
; CHECK-SD-NEXT: Ltmp8: ; EH_LABEL
-; CHECK-SD-NEXT: mov x1, x19
-; CHECK-SD-NEXT: mov x19, x0
-; CHECK-SD-NEXT: mov x0, x1
+; CHECK-SD-NEXT: str x0, [sp] ; 8-byte Spill
+; CHECK-SD-NEXT: mov x0, x19
; CHECK-SD-NEXT: bl _objc_release
; CHECK-SD-NEXT: b LBB4_5
; CHECK-SD-NEXT: LBB4_4: ; %lpad
; CHECK-SD-NEXT: Ltmp5: ; EH_LABEL
-; CHECK-SD-NEXT: mov x19, x0
+; CHECK-SD-NEXT: str x0, [sp] ; 8-byte Spill
; CHECK-SD-NEXT: LBB4_5: ; %ehcleanup
; CHECK-SD-NEXT: add x0, sp, #15
; CHECK-SD-NEXT: bl __ZN1SD1Ev
-; CHECK-SD-NEXT: mov x0, x19
+; CHECK-SD-NEXT: ldr x0, [sp] ; 8-byte Reload
; CHECK-SD-NEXT: bl __Unwind_Resume
; CHECK-SD-NEXT: Lfunc_end1:
; CHECK-SD-NEXT: .cfi_endproc
@@ -340,19 +343,19 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 {
; CHECK-GI-NEXT: Ltmp8: ; EH_LABEL
; CHECK-GI-NEXT: Lloh6:
; CHECK-GI-NEXT: adrp x8, _objc_release at GOTPAGE
-; CHECK-GI-NEXT: mov x20, x0
-; CHECK-GI-NEXT: mov x0, x19
; CHECK-GI-NEXT: Lloh7:
; CHECK-GI-NEXT: ldr x8, [x8, _objc_release at GOTPAGEOFF]
+; CHECK-GI-NEXT: str x0, [sp] ; 8-byte Spill
+; CHECK-GI-NEXT: mov x0, x19
; CHECK-GI-NEXT: blr x8
; CHECK-GI-NEXT: b LBB4_5
; CHECK-GI-NEXT: LBB4_4: ; %lpad
; CHECK-GI-NEXT: Ltmp5: ; EH_LABEL
-; CHECK-GI-NEXT: mov x20, x0
+; CHECK-GI-NEXT: str x0, [sp] ; 8-byte Spill
; CHECK-GI-NEXT: LBB4_5: ; %ehcleanup
; CHECK-GI-NEXT: add x0, sp, #15
; CHECK-GI-NEXT: bl __ZN1SD1Ev
-; CHECK-GI-NEXT: mov x0, x20
+; CHECK-GI-NEXT: ldr x0, [sp] ; 8-byte Reload
; CHECK-GI-NEXT: bl __Unwind_Resume
; CHECK-GI-NEXT: .loh AdrpLdrGot Lloh4, Lloh5
; CHECK-GI-NEXT: .loh AdrpLdrGot Lloh6, Lloh7
diff --git a/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll b/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
index c353a3e6496a8..a48c3e5e9b5c2 100644
--- a/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
+++ b/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
@@ -29,14 +29,12 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
; CHECK-NEXT: .cfi_personality 156, DW.ref.__gxx_personality_v0
; CHECK-NEXT: .cfi_lsda 28, .Lexception0
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT: str x19, [sp, #16] // 8-byte Spill
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: mov x29, sp
-; CHECK-NEXT: sub sp, sp, #16
-; CHECK-NEXT: .cfi_def_cfa w29, 32
-; CHECK-NEXT: .cfi_offset w19, -16
-; CHECK-NEXT: .cfi_offset w30, -24
-; CHECK-NEXT: .cfi_offset w29, -32
+; CHECK-NEXT: sub sp, sp, #32
+; CHECK-NEXT: .cfi_def_cfa w29, 16
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: rdsvl x8, #1
; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: msub x9, x8, x8, x9
@@ -45,8 +43,7 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
; CHECK-NEXT: tbnz w0, #0, .LBB0_2
; CHECK-NEXT: // %bb.1: // %return_normally
; CHECK-NEXT: mov sp, x29
-; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Reload
-; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: b shared_za_call
; CHECK-NEXT: .LBB0_2: // %throw_exception
; CHECK-NEXT: sub x8, x29, #16
@@ -73,7 +70,7 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
; CHECK-NEXT: // %bb.5: // %throw_fail
; CHECK-NEXT: .LBB0_6: // %unwind_dtors
; CHECK-NEXT: .Ltmp2: // EH_LABEL
-; CHECK-NEXT: mov x19, x0
+; CHECK-NEXT: stur x0, [x29, #-24] // 8-byte Folded Spill
; CHECK-NEXT: smstart za
; CHECK-NEXT: mrs x8, TPIDR2_EL0
; CHECK-NEXT: sub x0, x29, #16
@@ -84,8 +81,8 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
; CHECK-NEXT: msr TPIDR2_EL0, xzr
; CHECK-NEXT: bl shared_za_call
; CHECK-NEXT: sub x8, x29, #16
-; CHECK-NEXT: mov x0, x19
; CHECK-NEXT: msr TPIDR2_EL0, x8
+; CHECK-NEXT: ldur x0, [x29, #-24] // 8-byte Folded Reload
; CHECK-NEXT: bl _Unwind_Resume
br i1 %fail, label %throw_exception, label %return_normally
@@ -306,13 +303,13 @@ define void @try_catch_shared_zt0_callee() "aarch64_inout_zt0" personality ptr @
; CHECK-NEXT: .cfi_personality 156, DW.ref.__gxx_personality_v0
; CHECK-NEXT: .cfi_lsda 28, .Lexception3
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #80
-; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT: .cfi_def_cfa_offset 80
+; CHECK-NEXT: sub sp, sp, #96
+; CHECK-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: .cfi_offset w19, -8
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: .Ltmp9: // EH_LABEL
-; CHECK-NEXT: mov x19, sp
+; CHECK-NEXT: add x19, sp, #16
; CHECK-NEXT: str zt0, [x19]
; CHECK-NEXT: smstop za
; CHECK-NEXT: bl may_throw
@@ -320,20 +317,18 @@ define void @try_catch_shared_zt0_callee() "aarch64_inout_zt0" personality ptr @
; CHECK-NEXT: smstart za
; CHECK-NEXT: ldr zt0, [x19]
; CHECK-NEXT: // %bb.1: // %return_normally
-; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT: add sp, sp, #80
+; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT: add sp, sp, #96
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB3_2: // %unwind_dtors
; CHECK-NEXT: .Ltmp11: // EH_LABEL
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: mov x19, x0
+; CHECK-NEXT: str x0, [sp, #8] // 8-byte Spill
; CHECK-NEXT: smstart za
-; CHECK-NEXT: ldr zt0, [x8]
+; CHECK-NEXT: ldr zt0, [x19]
; CHECK-NEXT: bl shared_zt0_call
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: str zt0, [x8]
+; CHECK-NEXT: str zt0, [x19]
; CHECK-NEXT: smstop za
-; CHECK-NEXT: mov x0, x19
+; CHECK-NEXT: ldr x0, [sp, #8] // 8-byte Reload
; CHECK-NEXT: bl _Unwind_Resume
invoke void @may_throw()
to label %return_normally unwind label %unwind_dtors
@@ -601,7 +596,7 @@ define void @try_catch_shared_za_callee_zt0_saved(ptr %callee) "aarch64_inout_za
; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: mov x29, sp
-; CHECK-NEXT: sub sp, sp, #80
+; CHECK-NEXT: sub sp, sp, #96
; CHECK-NEXT: .cfi_def_cfa w29, 32
; CHECK-NEXT: .cfi_offset w19, -8
; CHECK-NEXT: .cfi_offset w20, -16
@@ -636,7 +631,7 @@ define void @try_catch_shared_za_callee_zt0_saved(ptr %callee) "aarch64_inout_za
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB8_4: // %unwind_dtors
; CHECK-NEXT: .Ltmp26: // EH_LABEL
-; CHECK-NEXT: mov x20, x0
+; CHECK-NEXT: stur x0, [x29, #-88] // 8-byte Folded Spill
; CHECK-NEXT: smstart za
; CHECK-NEXT: mrs x8, TPIDR2_EL0
; CHECK-NEXT: sub x0, x29, #80
@@ -647,8 +642,8 @@ define void @try_catch_shared_za_callee_zt0_saved(ptr %callee) "aarch64_inout_za
; CHECK-NEXT: msr TPIDR2_EL0, xzr
; CHECK-NEXT: blr x19
; CHECK-NEXT: sub x8, x29, #80
-; CHECK-NEXT: mov x0, x20
; CHECK-NEXT: msr TPIDR2_EL0, x8
+; CHECK-NEXT: ldur x0, [x29, #-88] // 8-byte Folded Reload
; CHECK-NEXT: bl _Unwind_Resume
invoke void @may_throw()
to label %return_normally unwind label %unwind_dtors
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 5da163b24d948..add1020821e6f 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -12352,22 +12352,16 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-LABEL: v_fadd_v32bf16:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
; GFX950-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT: v_add_f32_e32 v8, v8, v24
-; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT: v_add_f32_e32 v7, v7, v23
-; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT: v_add_f32_e32 v24, v8, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX950-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX950-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v30
@@ -12405,7 +12399,6 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_add_f32_e32 v26, v52, v51
; GFX950-NEXT: v_add_f32_e32 v9, v9, v25
; GFX950-NEXT: v_add_f32_e32 v25, v54, v53
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v8, v25
; GFX950-NEXT: v_cvt_pk_bf16_f32 v9, v9, v26
; GFX950-NEXT: v_cvt_pk_bf16_f32 v10, v10, v27
; GFX950-NEXT: v_cvt_pk_bf16_f32 v11, v11, v28
@@ -12413,50 +12406,55 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v13, v13, v30
; GFX950-NEXT: v_cvt_pk_bf16_f32 v14, v14, v33
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT: v_add_f32_e32 v24, v32, v24
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT: v_add_f32_e32 v32, v32, v8
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT: v_add_f32_e32 v15, v15, v31
+; GFX950-NEXT: v_add_f32_e32 v7, v7, v23
+; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT: v_add_f32_e32 v23, v32, v23
+; GFX950-NEXT: v_add_f32_e32 v23, v31, v23
; GFX950-NEXT: v_add_f32_e32 v6, v6, v22
; GFX950-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v5
; GFX950-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT: v_add_f32_e32 v22, v32, v22
+; GFX950-NEXT: v_add_f32_e32 v22, v31, v22
; GFX950-NEXT: v_add_f32_e32 v5, v5, v21
; GFX950-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v4
; GFX950-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT: v_add_f32_e32 v21, v32, v21
+; GFX950-NEXT: v_add_f32_e32 v21, v31, v21
; GFX950-NEXT: v_add_f32_e32 v4, v4, v20
; GFX950-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v3
; GFX950-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; GFX950-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT: v_add_f32_e32 v20, v32, v20
+; GFX950-NEXT: v_add_f32_e32 v20, v31, v20
; GFX950-NEXT: v_add_f32_e32 v3, v3, v19
; GFX950-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v2
; GFX950-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_add_f32_e32 v19, v32, v19
+; GFX950-NEXT: v_add_f32_e32 v19, v31, v19
; GFX950-NEXT: v_add_f32_e32 v2, v2, v18
; GFX950-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
; GFX950-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT: v_add_f32_e32 v18, v32, v18
+; GFX950-NEXT: v_add_f32_e32 v18, v31, v18
; GFX950-NEXT: v_add_f32_e32 v1, v1, v17
; GFX950-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
; GFX950-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT: v_add_f32_e32 v15, v15, v31
-; GFX950-NEXT: v_add_f32_e32 v31, v40, v55
-; GFX950-NEXT: v_add_f32_e32 v17, v32, v17
+; GFX950-NEXT: v_add_f32_e32 v8, v55, v8
+; GFX950-NEXT: v_add_f32_e32 v17, v31, v17
; GFX950-NEXT: v_add_f32_e32 v0, v0, v16
; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v17
; GFX950-NEXT: v_cvt_pk_bf16_f32 v1, v1, v18
@@ -12465,9 +12463,9 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v4, v4, v21
; GFX950-NEXT: v_cvt_pk_bf16_f32 v5, v5, v22
; GFX950-NEXT: v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v32
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fadd_v32bf16:
@@ -17521,22 +17519,16 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-LABEL: v_fmul_v32bf16:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
; GFX950-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT: v_mul_f32_e32 v8, v8, v24
-; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT: v_mul_f32_e32 v7, v7, v23
-; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT: v_mul_f32_e32 v24, v8, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX950-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX950-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v30
@@ -17574,7 +17566,6 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_mul_f32_e32 v26, v52, v51
; GFX950-NEXT: v_mul_f32_e32 v9, v9, v25
; GFX950-NEXT: v_mul_f32_e32 v25, v54, v53
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v8, v25
; GFX950-NEXT: v_cvt_pk_bf16_f32 v9, v9, v26
; GFX950-NEXT: v_cvt_pk_bf16_f32 v10, v10, v27
; GFX950-NEXT: v_cvt_pk_bf16_f32 v11, v11, v28
@@ -17582,50 +17573,55 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v13, v13, v30
; GFX950-NEXT: v_cvt_pk_bf16_f32 v14, v14, v33
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT: v_mul_f32_e32 v24, v32, v24
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT: v_mul_f32_e32 v32, v32, v8
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT: v_mul_f32_e32 v15, v15, v31
+; GFX950-NEXT: v_mul_f32_e32 v7, v7, v23
+; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT: v_mul_f32_e32 v23, v32, v23
+; GFX950-NEXT: v_mul_f32_e32 v23, v31, v23
; GFX950-NEXT: v_mul_f32_e32 v6, v6, v22
; GFX950-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v5
; GFX950-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT: v_mul_f32_e32 v22, v32, v22
+; GFX950-NEXT: v_mul_f32_e32 v22, v31, v22
; GFX950-NEXT: v_mul_f32_e32 v5, v5, v21
; GFX950-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v4
; GFX950-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT: v_mul_f32_e32 v21, v32, v21
+; GFX950-NEXT: v_mul_f32_e32 v21, v31, v21
; GFX950-NEXT: v_mul_f32_e32 v4, v4, v20
; GFX950-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v3
; GFX950-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; GFX950-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT: v_mul_f32_e32 v20, v32, v20
+; GFX950-NEXT: v_mul_f32_e32 v20, v31, v20
; GFX950-NEXT: v_mul_f32_e32 v3, v3, v19
; GFX950-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v2
; GFX950-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_mul_f32_e32 v19, v32, v19
+; GFX950-NEXT: v_mul_f32_e32 v19, v31, v19
; GFX950-NEXT: v_mul_f32_e32 v2, v2, v18
; GFX950-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
; GFX950-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT: v_mul_f32_e32 v18, v32, v18
+; GFX950-NEXT: v_mul_f32_e32 v18, v31, v18
; GFX950-NEXT: v_mul_f32_e32 v1, v1, v17
; GFX950-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
; GFX950-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT: v_mul_f32_e32 v15, v15, v31
-; GFX950-NEXT: v_mul_f32_e32 v31, v40, v55
-; GFX950-NEXT: v_mul_f32_e32 v17, v32, v17
+; GFX950-NEXT: v_mul_f32_e32 v8, v55, v8
+; GFX950-NEXT: v_mul_f32_e32 v17, v31, v17
; GFX950-NEXT: v_mul_f32_e32 v0, v0, v16
; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v17
; GFX950-NEXT: v_cvt_pk_bf16_f32 v1, v1, v18
@@ -17634,9 +17630,9 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v4, v4, v21
; GFX950-NEXT: v_cvt_pk_bf16_f32 v5, v5, v22
; GFX950-NEXT: v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v32
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v32bf16:
@@ -22516,22 +22512,16 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-LABEL: v_minnum_v32bf16:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
; GFX950-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT: v_min_f32_e32 v8, v8, v24
-; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT: v_min_f32_e32 v7, v7, v23
-; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT: v_min_f32_e32 v24, v8, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX950-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX950-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v30
@@ -22569,7 +22559,6 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_min_f32_e32 v26, v52, v51
; GFX950-NEXT: v_min_f32_e32 v9, v9, v25
; GFX950-NEXT: v_min_f32_e32 v25, v54, v53
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v8, v25
; GFX950-NEXT: v_cvt_pk_bf16_f32 v9, v9, v26
; GFX950-NEXT: v_cvt_pk_bf16_f32 v10, v10, v27
; GFX950-NEXT: v_cvt_pk_bf16_f32 v11, v11, v28
@@ -22577,50 +22566,55 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v13, v13, v30
; GFX950-NEXT: v_cvt_pk_bf16_f32 v14, v14, v33
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT: v_min_f32_e32 v24, v32, v24
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT: v_min_f32_e32 v32, v32, v8
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT: v_min_f32_e32 v15, v15, v31
+; GFX950-NEXT: v_min_f32_e32 v7, v7, v23
+; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT: v_min_f32_e32 v23, v32, v23
+; GFX950-NEXT: v_min_f32_e32 v23, v31, v23
; GFX950-NEXT: v_min_f32_e32 v6, v6, v22
; GFX950-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v5
; GFX950-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT: v_min_f32_e32 v22, v32, v22
+; GFX950-NEXT: v_min_f32_e32 v22, v31, v22
; GFX950-NEXT: v_min_f32_e32 v5, v5, v21
; GFX950-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v4
; GFX950-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT: v_min_f32_e32 v21, v32, v21
+; GFX950-NEXT: v_min_f32_e32 v21, v31, v21
; GFX950-NEXT: v_min_f32_e32 v4, v4, v20
; GFX950-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v3
; GFX950-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; GFX950-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v20, v32, v20
+; GFX950-NEXT: v_min_f32_e32 v20, v31, v20
; GFX950-NEXT: v_min_f32_e32 v3, v3, v19
; GFX950-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v2
; GFX950-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_min_f32_e32 v19, v32, v19
+; GFX950-NEXT: v_min_f32_e32 v19, v31, v19
; GFX950-NEXT: v_min_f32_e32 v2, v2, v18
; GFX950-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
; GFX950-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT: v_min_f32_e32 v18, v32, v18
+; GFX950-NEXT: v_min_f32_e32 v18, v31, v18
; GFX950-NEXT: v_min_f32_e32 v1, v1, v17
; GFX950-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
; GFX950-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT: v_min_f32_e32 v15, v15, v31
-; GFX950-NEXT: v_min_f32_e32 v31, v40, v55
-; GFX950-NEXT: v_min_f32_e32 v17, v32, v17
+; GFX950-NEXT: v_min_f32_e32 v8, v55, v8
+; GFX950-NEXT: v_min_f32_e32 v17, v31, v17
; GFX950-NEXT: v_min_f32_e32 v0, v0, v16
; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v17
; GFX950-NEXT: v_cvt_pk_bf16_f32 v1, v1, v18
@@ -22629,9 +22623,9 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v4, v4, v21
; GFX950-NEXT: v_cvt_pk_bf16_f32 v5, v5, v22
; GFX950-NEXT: v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v32
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_minnum_v32bf16:
@@ -26943,22 +26937,16 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-LABEL: v_maxnum_v32bf16:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
; GFX950-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT: v_max_f32_e32 v8, v8, v24
-; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT: v_max_f32_e32 v7, v7, v23
-; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT: v_max_f32_e32 v24, v8, v24
; GFX950-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX950-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX950-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v30
@@ -26996,7 +26984,6 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_max_f32_e32 v26, v52, v51
; GFX950-NEXT: v_max_f32_e32 v9, v9, v25
; GFX950-NEXT: v_max_f32_e32 v25, v54, v53
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v8, v25
; GFX950-NEXT: v_cvt_pk_bf16_f32 v9, v9, v26
; GFX950-NEXT: v_cvt_pk_bf16_f32 v10, v10, v27
; GFX950-NEXT: v_cvt_pk_bf16_f32 v11, v11, v28
@@ -27004,50 +26991,55 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v13, v13, v30
; GFX950-NEXT: v_cvt_pk_bf16_f32 v14, v14, v33
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT: v_max_f32_e32 v24, v32, v24
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT: v_max_f32_e32 v32, v32, v8
+; GFX950-NEXT: v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT: v_max_f32_e32 v15, v15, v31
+; GFX950-NEXT: v_max_f32_e32 v7, v7, v23
+; GFX950-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT: v_max_f32_e32 v23, v32, v23
+; GFX950-NEXT: v_max_f32_e32 v23, v31, v23
; GFX950-NEXT: v_max_f32_e32 v6, v6, v22
; GFX950-NEXT: v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v5
; GFX950-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT: v_max_f32_e32 v22, v32, v22
+; GFX950-NEXT: v_max_f32_e32 v22, v31, v22
; GFX950-NEXT: v_max_f32_e32 v5, v5, v21
; GFX950-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v4
; GFX950-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT: v_max_f32_e32 v21, v32, v21
+; GFX950-NEXT: v_max_f32_e32 v21, v31, v21
; GFX950-NEXT: v_max_f32_e32 v4, v4, v20
; GFX950-NEXT: v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v3
; GFX950-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; GFX950-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v20, v32, v20
+; GFX950-NEXT: v_max_f32_e32 v20, v31, v20
; GFX950-NEXT: v_max_f32_e32 v3, v3, v19
; GFX950-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v2
; GFX950-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_max_f32_e32 v19, v32, v19
+; GFX950-NEXT: v_max_f32_e32 v19, v31, v19
; GFX950-NEXT: v_max_f32_e32 v2, v2, v18
; GFX950-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
; GFX950-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT: v_max_f32_e32 v18, v32, v18
+; GFX950-NEXT: v_max_f32_e32 v18, v31, v18
; GFX950-NEXT: v_max_f32_e32 v1, v1, v17
; GFX950-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT: v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
; GFX950-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT: v_max_f32_e32 v15, v15, v31
-; GFX950-NEXT: v_max_f32_e32 v31, v40, v55
-; GFX950-NEXT: v_max_f32_e32 v17, v32, v17
+; GFX950-NEXT: v_max_f32_e32 v8, v55, v8
+; GFX950-NEXT: v_max_f32_e32 v17, v31, v17
; GFX950-NEXT: v_max_f32_e32 v0, v0, v16
; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v17
; GFX950-NEXT: v_cvt_pk_bf16_f32 v1, v1, v18
@@ -27056,9 +27048,9 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX950-NEXT: v_cvt_pk_bf16_f32 v4, v4, v21
; GFX950-NEXT: v_cvt_pk_bf16_f32 v5, v5, v22
; GFX950-NEXT: v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v15, v32
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_maxnum_v32bf16:
@@ -48752,7 +48744,6 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX950-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX950-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
; GFX950-NEXT: scratch_load_dword v31, off, s32 offset:64
; GFX950-NEXT: scratch_load_dword v32, off, s32
; GFX950-NEXT: scratch_load_dword v33, off, s32 offset:60
@@ -48768,45 +48759,47 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX950-NEXT: scratch_load_dword v51, off, s32 offset:20
; GFX950-NEXT: scratch_load_dword v52, off, s32 offset:16
; GFX950-NEXT: scratch_load_dword v53, off, s32 offset:12
-; GFX950-NEXT: scratch_load_dword v54, off, s32 offset:8
+; GFX950-NEXT: scratch_load_dword v58, off, s32 offset:8
; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v15
-; GFX950-NEXT: v_and_b32_e32 v42, 0xffff0000, v29
-; GFX950-NEXT: v_and_b32_e32 v43, 0xffff0000, v13
-; GFX950-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX950-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GFX950-NEXT: v_and_b32_e32 v44, 0xffff0000, v28
; GFX950-NEXT: v_and_b32_e32 v45, 0xffff0000, v12
; GFX950-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; GFX950-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT: v_and_b32_e32 v47, 0xffff0000, v11
+; GFX950-NEXT: v_and_b32_e32 v42, 0xffff0000, v29
+; GFX950-NEXT: v_and_b32_e32 v43, 0xffff0000, v13
+; GFX950-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX950-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX950-NEXT: v_and_b32_e32 v57, 0xffff0000, v11
; GFX950-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GFX950-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v30
; GFX950-NEXT: v_and_b32_e32 v41, 0xffff0000, v14
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX950-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX950-NEXT: v_and_b32_e32 v46, 0xffff0000, v27
; GFX950-NEXT: s_waitcnt vmcnt(15)
-; GFX950-NEXT: v_and_b32_e32 v56, 0xffff0000, v31
+; GFX950-NEXT: v_and_b32_e32 v46, 0xffff0000, v31
; GFX950-NEXT: s_waitcnt vmcnt(14)
-; GFX950-NEXT: v_and_b32_e32 v57, 0xffff0000, v32
-; GFX950-NEXT: v_fmac_f32_e32 v56, v55, v57
+; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v32
+; GFX950-NEXT: v_fmac_f32_e32 v46, v55, v54
; GFX950-NEXT: s_waitcnt vmcnt(12)
-; GFX950-NEXT: v_and_b32_e32 v59, 0xffff0000, v34
-; GFX950-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX950-NEXT: v_and_b32_e32 v56, 0xffff0000, v34
; GFX950-NEXT: s_waitcnt vmcnt(11)
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v35
+; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v35
; GFX950-NEXT: v_lshlrev_b32_e32 v35, 16, v35
-; GFX950-NEXT: v_fmac_f32_e32 v34, v13, v29
-; GFX950-NEXT: v_fmac_f32_e32 v35, v12, v28
+; GFX950-NEXT: v_lshlrev_b32_e32 v34, 16, v34
; GFX950-NEXT: s_waitcnt vmcnt(10)
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v36
+; GFX950-NEXT: v_fmac_f32_e32 v35, v12, v28
+; GFX950-NEXT: v_and_b32_e32 v12, 0xffff0000, v27
+; GFX950-NEXT: v_fmac_f32_e32 v34, v13, v29
+; GFX950-NEXT: v_fmac_f32_e32 v55, v57, v12
; GFX950-NEXT: v_lshlrev_b32_e32 v12, 16, v36
; GFX950-NEXT: v_lshlrev_b32_e32 v13, 16, v27
; GFX950-NEXT: v_fmac_f32_e32 v12, v11, v13
; GFX950-NEXT: scratch_load_dword v13, off, s32 offset:4
; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; GFX950-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; GFX950-NEXT: v_and_b32_e32 v58, 0xffff0000, v33
+; GFX950-NEXT: v_and_b32_e32 v47, 0xffff0000, v33
; GFX950-NEXT: v_lshlrev_b32_e32 v33, 16, v33
; GFX950-NEXT: v_fmac_f32_e32 v31, v15, v32
; GFX950-NEXT: v_fmac_f32_e32 v33, v14, v30
@@ -48892,14 +48885,13 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX950-NEXT: v_fmac_f32_e32 v19, v2, v18
; GFX950-NEXT: s_waitcnt vmcnt(1)
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v54
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v58
; GFX950-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
; GFX950-NEXT: v_and_b32_e32 v26, 0xffff0000, v1
; GFX950-NEXT: v_fmac_f32_e32 v2, v26, v18
-; GFX950-NEXT: v_lshlrev_b32_e32 v18, 16, v54
+; GFX950-NEXT: v_lshlrev_b32_e32 v18, 16, v58
; GFX950-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_and_b32_e32 v57, 0xffff0000, v36
; GFX950-NEXT: v_fmac_f32_e32 v18, v1, v17
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
@@ -48908,10 +48900,9 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX950-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GFX950-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT: v_fmac_f32_e32 v58, v41, v40
-; GFX950-NEXT: v_fmac_f32_e32 v59, v43, v42
-; GFX950-NEXT: v_fmac_f32_e32 v55, v45, v44
-; GFX950-NEXT: v_fmac_f32_e32 v57, v47, v46
+; GFX950-NEXT: v_fmac_f32_e32 v47, v41, v40
+; GFX950-NEXT: v_fmac_f32_e32 v56, v43, v42
+; GFX950-NEXT: v_fmac_f32_e32 v54, v45, v44
; GFX950-NEXT: v_fmac_f32_e32 v1, v26, v17
; GFX950-NEXT: v_fmac_f32_e32 v13, v0, v16
; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v13, v1
@@ -48925,12 +48916,11 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX950-NEXT: v_cvt_pk_bf16_f32 v8, v25, v9
; GFX950-NEXT: v_cvt_pk_bf16_f32 v9, v15, v10
; GFX950-NEXT: v_cvt_pk_bf16_f32 v10, v14, v11
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v11, v12, v57
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v12, v35, v55
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v13, v34, v59
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v14, v33, v58
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v31, v56
-; GFX950-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v11, v12, v55
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v12, v35, v54
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v13, v34, v56
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v14, v33, v47
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v15, v31, v46
; GFX950-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
index 918fa1364d853..4bda91924871c 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
@@ -2217,99 +2217,86 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
; GCN-LABEL: v_copysign_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GCN-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GCN-NEXT: s_waitcnt expcnt(1)
-; GCN-NEXT: v_and_b32_e32 v58, 0x8000, v17
-; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT: v_and_b32_e32 v33, 0x7fff, v14
+; GCN-NEXT: s_waitcnt expcnt(2)
+; GCN-NEXT: v_and_b32_e32 v40, 0x8000, v30
+; GCN-NEXT: v_or_b32_e32 v37, v33, v40
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v29
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v13
+; GCN-NEXT: v_or_b32_e32 v39, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v28
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v12
+; GCN-NEXT: v_or_b32_e32 v50, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v27
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v11
+; GCN-NEXT: v_or_b32_e32 v51, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v26
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v10
+; GCN-NEXT: v_or_b32_e32 v54, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v25
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v9
+; GCN-NEXT: v_or_b32_e32 v55, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v24
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v8
; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: v_and_b32_e32 v59, 0x7fff, v1
-; GCN-NEXT: v_and_b32_e32 v17, 0x8000, v17
-; GCN-NEXT: v_bfe_u32 v1, v1, 16, 15
-; GCN-NEXT: v_and_b32_e32 v38, 0x8000, v27
-; GCN-NEXT: v_and_b32_e32 v39, 0x7fff, v11
-; GCN-NEXT: v_and_b32_e32 v48, 0x8000, v26
-; GCN-NEXT: v_and_b32_e32 v49, 0x7fff, v10
-; GCN-NEXT: v_and_b32_e32 v50, 0x8000, v25
-; GCN-NEXT: v_and_b32_e32 v51, 0x7fff, v9
-; GCN-NEXT: v_and_b32_e32 v40, 0x8000, v22
-; GCN-NEXT: v_and_b32_e32 v41, 0x7fff, v6
-; GCN-NEXT: v_or_b32_e32 v1, v1, v17
-; GCN-NEXT: v_and_b32_e32 v52, 0x8000, v24
-; GCN-NEXT: v_and_b32_e32 v53, 0x7fff, v8
-; GCN-NEXT: v_and_b32_e32 v54, 0x8000, v23
-; GCN-NEXT: v_and_b32_e32 v55, 0x7fff, v7
-; GCN-NEXT: v_and_b32_e32 v42, 0x8000, v21
-; GCN-NEXT: v_and_b32_e32 v43, 0x7fff, v5
-; GCN-NEXT: v_and_b32_e32 v44, 0x8000, v20
-; GCN-NEXT: v_and_b32_e32 v45, 0x7fff, v4
-; GCN-NEXT: v_and_b32_e32 v46, 0x8000, v19
-; GCN-NEXT: v_and_b32_e32 v47, 0x7fff, v3
-; GCN-NEXT: v_and_b32_e32 v56, 0x8000, v18
-; GCN-NEXT: v_and_b32_e32 v57, 0x7fff, v2
-; GCN-NEXT: v_or_b32_e32 v38, v39, v38
-; GCN-NEXT: v_or_b32_e32 v39, v49, v48
-; GCN-NEXT: v_or_b32_e32 v48, v51, v50
-; GCN-NEXT: v_or_b32_e32 v51, v41, v40
-; GCN-NEXT: v_or_b32_e32 v40, v59, v58
-; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_or_b32_e32 v49, v53, v52
-; GCN-NEXT: v_or_b32_e32 v50, v55, v54
-; GCN-NEXT: v_or_b32_e32 v52, v43, v42
-; GCN-NEXT: v_or_b32_e32 v53, v45, v44
-; GCN-NEXT: v_or_b32_e32 v54, v47, v46
-; GCN-NEXT: v_or_b32_e32 v55, v57, v56
-; GCN-NEXT: v_or_b32_e32 v1, v40, v1
-; GCN-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT: v_or_b32_e32 v42, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v23
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v7
+; GCN-NEXT: v_or_b32_e32 v32, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v22
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v6
+; GCN-NEXT: v_or_b32_e32 v34, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v21
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v5
; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GCN-NEXT: v_and_b32_e32 v32, 0x8000, v30
-; GCN-NEXT: v_and_b32_e32 v33, 0x7fff, v14
-; GCN-NEXT: v_and_b32_e32 v34, 0x8000, v29
-; GCN-NEXT: v_and_b32_e32 v35, 0x7fff, v13
-; GCN-NEXT: v_or_b32_e32 v32, v33, v32
-; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v16
-; GCN-NEXT: v_or_b32_e32 v34, v35, v34
-; GCN-NEXT: v_and_b32_e32 v35, 0x7fff, v0
-; GCN-NEXT: v_and_b32_e32 v36, 0x8000, v28
-; GCN-NEXT: v_and_b32_e32 v37, 0x7fff, v12
-; GCN-NEXT: v_or_b32_e32 v33, v35, v33
+; GCN-NEXT: v_or_b32_e32 v35, v40, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v20
+; GCN-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GCN-NEXT: v_and_b32_e32 v40, 0x7fff, v4
+; GCN-NEXT: v_and_b32_e32 v24, 0x8000, v24
+; GCN-NEXT: v_bfe_u32 v8, v8, 16, 15
+; GCN-NEXT: v_and_b32_e32 v20, 0x8000, v20
+; GCN-NEXT: v_bfe_u32 v4, v4, 16, 15
+; GCN-NEXT: v_or_b32_e32 v8, v8, v24
+; GCN-NEXT: v_or_b32_e32 v4, v4, v20
+; GCN-NEXT: v_and_b32_e32 v36, 0x8000, v17
+; GCN-NEXT: v_and_b32_e32 v38, 0x7fff, v1
+; GCN-NEXT: v_and_b32_e32 v41, 0x8000, v16
+; GCN-NEXT: v_and_b32_e32 v52, 0x7fff, v0
+; GCN-NEXT: v_or_b32_e32 v40, v40, v33
+; GCN-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_or_b32_e32 v36, v38, v36
+; GCN-NEXT: v_or_b32_e32 v38, v52, v41
+; GCN-NEXT: v_or_b32_e32 v4, v40, v4
+; GCN-NEXT: v_or_b32_e32 v8, v42, v8
+; GCN-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT: v_and_b32_e32 v49, 0x7fff, v3
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v19
+; GCN-NEXT: v_and_b32_e32 v48, 0x7fff, v2
+; GCN-NEXT: v_or_b32_e32 v49, v49, v33
+; GCN-NEXT: v_and_b32_e32 v33, 0x8000, v18
+; GCN-NEXT: v_or_b32_e32 v33, v48, v33
; GCN-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; GCN-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; GCN-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; GCN-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; GCN-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; GCN-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; GCN-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; GCN-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; GCN-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; GCN-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; GCN-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GCN-NEXT: v_or_b32_e32 v36, v37, v36
-; GCN-NEXT: v_and_b32_e32 v37, 0x7fff, v15
+; GCN-NEXT: v_and_b32_e32 v53, 0x7fff, v15
; GCN-NEXT: v_bfe_u32 v15, v15, 16, 15
; GCN-NEXT: v_and_b32_e32 v30, 0x8000, v30
; GCN-NEXT: v_bfe_u32 v14, v14, 16, 15
@@ -2323,20 +2310,18 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
; GCN-NEXT: v_bfe_u32 v10, v10, 16, 15
; GCN-NEXT: v_and_b32_e32 v25, 0x8000, v25
; GCN-NEXT: v_bfe_u32 v9, v9, 16, 15
-; GCN-NEXT: v_and_b32_e32 v24, 0x8000, v24
-; GCN-NEXT: v_bfe_u32 v8, v8, 16, 15
; GCN-NEXT: v_and_b32_e32 v23, 0x8000, v23
; GCN-NEXT: v_bfe_u32 v7, v7, 16, 15
; GCN-NEXT: v_and_b32_e32 v22, 0x8000, v22
; GCN-NEXT: v_bfe_u32 v6, v6, 16, 15
; GCN-NEXT: v_and_b32_e32 v21, 0x8000, v21
; GCN-NEXT: v_bfe_u32 v5, v5, 16, 15
-; GCN-NEXT: v_and_b32_e32 v20, 0x8000, v20
-; GCN-NEXT: v_bfe_u32 v4, v4, 16, 15
; GCN-NEXT: v_and_b32_e32 v19, 0x8000, v19
; GCN-NEXT: v_bfe_u32 v3, v3, 16, 15
; GCN-NEXT: v_and_b32_e32 v18, 0x8000, v18
; GCN-NEXT: v_bfe_u32 v2, v2, 16, 15
+; GCN-NEXT: v_and_b32_e32 v17, 0x8000, v17
+; GCN-NEXT: v_bfe_u32 v1, v1, 16, 15
; GCN-NEXT: v_and_b32_e32 v16, 0x8000, v16
; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
; GCN-NEXT: v_or_b32_e32 v14, v14, v30
@@ -2345,146 +2330,131 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
; GCN-NEXT: v_or_b32_e32 v11, v11, v27
; GCN-NEXT: v_or_b32_e32 v10, v10, v26
; GCN-NEXT: v_or_b32_e32 v9, v9, v25
-; GCN-NEXT: v_or_b32_e32 v8, v8, v24
; GCN-NEXT: v_or_b32_e32 v7, v7, v23
; GCN-NEXT: v_or_b32_e32 v6, v6, v22
; GCN-NEXT: v_or_b32_e32 v5, v5, v21
-; GCN-NEXT: v_or_b32_e32 v4, v4, v20
+; GCN-NEXT: s_waitcnt vmcnt(3)
+; GCN-NEXT: v_and_b32_e32 v48, 0x8000, v31
+; GCN-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; GCN-NEXT: v_and_b32_e32 v31, 0x8000, v31
+; GCN-NEXT: v_or_b32_e32 v15, v15, v31
; GCN-NEXT: v_or_b32_e32 v3, v3, v19
; GCN-NEXT: v_or_b32_e32 v2, v2, v18
+; GCN-NEXT: v_or_b32_e32 v1, v1, v17
; GCN-NEXT: v_or_b32_e32 v0, v0, v16
+; GCN-NEXT: v_or_b32_e32 v48, v53, v48
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GCN-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GCN-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GCN-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GCN-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v35, 0x8000, v31
-; GCN-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; GCN-NEXT: v_and_b32_e32 v31, 0x8000, v31
-; GCN-NEXT: v_or_b32_e32 v15, v15, v31
-; GCN-NEXT: v_or_b32_e32 v35, v37, v35
; GCN-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GCN-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GCN-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GCN-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GCN-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GCN-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GCN-NEXT: v_or_b32_e32 v0, v33, v0
-; GCN-NEXT: v_or_b32_e32 v2, v55, v2
-; GCN-NEXT: v_or_b32_e32 v3, v54, v3
-; GCN-NEXT: v_or_b32_e32 v4, v53, v4
-; GCN-NEXT: v_or_b32_e32 v5, v52, v5
-; GCN-NEXT: v_or_b32_e32 v6, v51, v6
-; GCN-NEXT: v_or_b32_e32 v7, v50, v7
-; GCN-NEXT: v_or_b32_e32 v8, v49, v8
-; GCN-NEXT: v_or_b32_e32 v9, v48, v9
-; GCN-NEXT: v_or_b32_e32 v10, v39, v10
-; GCN-NEXT: v_or_b32_e32 v11, v38, v11
-; GCN-NEXT: v_or_b32_e32 v12, v36, v12
-; GCN-NEXT: v_or_b32_e32 v13, v34, v13
-; GCN-NEXT: v_or_b32_e32 v14, v32, v14
-; GCN-NEXT: v_or_b32_e32 v15, v35, v15
+; GCN-NEXT: v_or_b32_e32 v0, v38, v0
+; GCN-NEXT: v_or_b32_e32 v1, v36, v1
+; GCN-NEXT: v_or_b32_e32 v2, v33, v2
+; GCN-NEXT: v_or_b32_e32 v3, v49, v3
+; GCN-NEXT: v_or_b32_e32 v5, v35, v5
+; GCN-NEXT: v_or_b32_e32 v6, v34, v6
+; GCN-NEXT: v_or_b32_e32 v7, v32, v7
+; GCN-NEXT: v_or_b32_e32 v9, v55, v9
+; GCN-NEXT: v_or_b32_e32 v10, v54, v10
+; GCN-NEXT: v_or_b32_e32 v11, v51, v11
+; GCN-NEXT: v_or_b32_e32 v12, v50, v12
+; GCN-NEXT: v_or_b32_e32 v13, v39, v13
+; GCN-NEXT: v_or_b32_e32 v14, v37, v14
+; GCN-NEXT: v_or_b32_e32 v15, v48, v15
+; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_copysign_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX7-NEXT: v_and_b32_e32 v58, 0x8000, v17
-; GFX7-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_and_b32_e32 v59, 0x7fff, v1
-; GFX7-NEXT: v_and_b32_e32 v17, 0x8000, v17
-; GFX7-NEXT: v_bfe_u32 v1, v1, 16, 15
-; GFX7-NEXT: v_and_b32_e32 v38, 0x8000, v27
-; GFX7-NEXT: v_and_b32_e32 v39, 0x7fff, v11
-; GFX7-NEXT: v_and_b32_e32 v48, 0x8000, v26
-; GFX7-NEXT: v_and_b32_e32 v49, 0x7fff, v10
-; GFX7-NEXT: v_and_b32_e32 v50, 0x8000, v25
-; GFX7-NEXT: v_and_b32_e32 v51, 0x7fff, v9
-; GFX7-NEXT: v_and_b32_e32 v40, 0x8000, v22
-; GFX7-NEXT: v_and_b32_e32 v41, 0x7fff, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
-; GFX7-NEXT: v_and_b32_e32 v52, 0x8000, v24
-; GFX7-NEXT: v_and_b32_e32 v53, 0x7fff, v8
-; GFX7-NEXT: v_and_b32_e32 v54, 0x8000, v23
-; GFX7-NEXT: v_and_b32_e32 v55, 0x7fff, v7
-; GFX7-NEXT: v_and_b32_e32 v42, 0x8000, v21
-; GFX7-NEXT: v_and_b32_e32 v43, 0x7fff, v5
-; GFX7-NEXT: v_and_b32_e32 v44, 0x8000, v20
-; GFX7-NEXT: v_and_b32_e32 v45, 0x7fff, v4
-; GFX7-NEXT: v_and_b32_e32 v46, 0x8000, v19
-; GFX7-NEXT: v_and_b32_e32 v47, 0x7fff, v3
-; GFX7-NEXT: v_and_b32_e32 v56, 0x8000, v18
-; GFX7-NEXT: v_and_b32_e32 v57, 0x7fff, v2
-; GFX7-NEXT: v_or_b32_e32 v38, v39, v38
-; GFX7-NEXT: v_or_b32_e32 v39, v49, v48
-; GFX7-NEXT: v_or_b32_e32 v48, v51, v50
-; GFX7-NEXT: v_or_b32_e32 v51, v41, v40
-; GFX7-NEXT: v_or_b32_e32 v40, v59, v58
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v49, v53, v52
-; GFX7-NEXT: v_or_b32_e32 v50, v55, v54
-; GFX7-NEXT: v_or_b32_e32 v52, v43, v42
-; GFX7-NEXT: v_or_b32_e32 v53, v45, v44
-; GFX7-NEXT: v_or_b32_e32 v54, v47, v46
-; GFX7-NEXT: v_or_b32_e32 v55, v57, v56
-; GFX7-NEXT: v_or_b32_e32 v1, v40, v1
-; GFX7-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX7-NEXT: v_and_b32_e32 v32, 0x8000, v30
+; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX7-NEXT: v_and_b32_e32 v33, 0x7fff, v14
-; GFX7-NEXT: v_and_b32_e32 v34, 0x8000, v29
-; GFX7-NEXT: v_and_b32_e32 v35, 0x7fff, v13
-; GFX7-NEXT: v_or_b32_e32 v32, v33, v32
-; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v16
-; GFX7-NEXT: v_or_b32_e32 v34, v35, v34
-; GFX7-NEXT: v_and_b32_e32 v35, 0x7fff, v0
-; GFX7-NEXT: v_and_b32_e32 v36, 0x8000, v28
-; GFX7-NEXT: v_and_b32_e32 v37, 0x7fff, v12
-; GFX7-NEXT: v_or_b32_e32 v33, v35, v33
+; GFX7-NEXT: v_and_b32_e32 v40, 0x8000, v30
+; GFX7-NEXT: v_or_b32_e32 v37, v33, v40
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v29
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v13
+; GFX7-NEXT: v_or_b32_e32 v39, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v28
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v12
+; GFX7-NEXT: v_or_b32_e32 v50, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v27
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v11
+; GFX7-NEXT: v_or_b32_e32 v51, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v26
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v10
+; GFX7-NEXT: v_or_b32_e32 v54, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v25
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v9
+; GFX7-NEXT: v_or_b32_e32 v55, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v24
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v8
+; GFX7-NEXT: v_or_b32_e32 v42, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v23
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v7
+; GFX7-NEXT: v_or_b32_e32 v32, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v22
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v6
+; GFX7-NEXT: v_or_b32_e32 v34, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v21
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v5
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX7-NEXT: v_or_b32_e32 v35, v40, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v20
+; GFX7-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GFX7-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GFX7-NEXT: v_and_b32_e32 v40, 0x7fff, v4
+; GFX7-NEXT: v_and_b32_e32 v24, 0x8000, v24
+; GFX7-NEXT: v_bfe_u32 v8, v8, 16, 15
+; GFX7-NEXT: v_and_b32_e32 v20, 0x8000, v20
+; GFX7-NEXT: v_bfe_u32 v4, v4, 16, 15
+; GFX7-NEXT: v_or_b32_e32 v8, v8, v24
+; GFX7-NEXT: v_or_b32_e32 v4, v4, v20
+; GFX7-NEXT: v_and_b32_e32 v36, 0x8000, v17
+; GFX7-NEXT: v_and_b32_e32 v38, 0x7fff, v1
+; GFX7-NEXT: v_and_b32_e32 v41, 0x8000, v16
+; GFX7-NEXT: v_and_b32_e32 v52, 0x7fff, v0
+; GFX7-NEXT: v_or_b32_e32 v40, v40, v33
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_or_b32_e32 v36, v38, v36
+; GFX7-NEXT: v_or_b32_e32 v38, v52, v41
+; GFX7-NEXT: v_or_b32_e32 v4, v40, v4
+; GFX7-NEXT: v_or_b32_e32 v8, v42, v8
+; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT: v_and_b32_e32 v49, 0x7fff, v3
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v19
+; GFX7-NEXT: v_and_b32_e32 v48, 0x7fff, v2
+; GFX7-NEXT: v_or_b32_e32 v49, v49, v33
+; GFX7-NEXT: v_and_b32_e32 v33, 0x8000, v18
+; GFX7-NEXT: v_or_b32_e32 v33, v48, v33
; GFX7-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; GFX7-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; GFX7-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; GFX7-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; GFX7-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; GFX7-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; GFX7-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; GFX7-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; GFX7-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; GFX7-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; GFX7-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX7-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GFX7-NEXT: v_or_b32_e32 v36, v37, v36
-; GFX7-NEXT: v_and_b32_e32 v37, 0x7fff, v15
+; GFX7-NEXT: v_and_b32_e32 v53, 0x7fff, v15
; GFX7-NEXT: v_bfe_u32 v15, v15, 16, 15
; GFX7-NEXT: v_and_b32_e32 v30, 0x8000, v30
; GFX7-NEXT: v_bfe_u32 v14, v14, 16, 15
@@ -2498,20 +2468,18 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
; GFX7-NEXT: v_bfe_u32 v10, v10, 16, 15
; GFX7-NEXT: v_and_b32_e32 v25, 0x8000, v25
; GFX7-NEXT: v_bfe_u32 v9, v9, 16, 15
-; GFX7-NEXT: v_and_b32_e32 v24, 0x8000, v24
-; GFX7-NEXT: v_bfe_u32 v8, v8, 16, 15
; GFX7-NEXT: v_and_b32_e32 v23, 0x8000, v23
; GFX7-NEXT: v_bfe_u32 v7, v7, 16, 15
; GFX7-NEXT: v_and_b32_e32 v22, 0x8000, v22
; GFX7-NEXT: v_bfe_u32 v6, v6, 16, 15
; GFX7-NEXT: v_and_b32_e32 v21, 0x8000, v21
; GFX7-NEXT: v_bfe_u32 v5, v5, 16, 15
-; GFX7-NEXT: v_and_b32_e32 v20, 0x8000, v20
-; GFX7-NEXT: v_bfe_u32 v4, v4, 16, 15
; GFX7-NEXT: v_and_b32_e32 v19, 0x8000, v19
; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 15
; GFX7-NEXT: v_and_b32_e32 v18, 0x8000, v18
; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 15
+; GFX7-NEXT: v_and_b32_e32 v17, 0x8000, v17
+; GFX7-NEXT: v_bfe_u32 v1, v1, 16, 15
; GFX7-NEXT: v_and_b32_e32 v16, 0x8000, v16
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7-NEXT: v_or_b32_e32 v14, v14, v30
@@ -2520,50 +2488,48 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
; GFX7-NEXT: v_or_b32_e32 v11, v11, v27
; GFX7-NEXT: v_or_b32_e32 v10, v10, v26
; GFX7-NEXT: v_or_b32_e32 v9, v9, v25
-; GFX7-NEXT: v_or_b32_e32 v8, v8, v24
; GFX7-NEXT: v_or_b32_e32 v7, v7, v23
; GFX7-NEXT: v_or_b32_e32 v6, v6, v22
; GFX7-NEXT: v_or_b32_e32 v5, v5, v21
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v20
+; GFX7-NEXT: s_waitcnt vmcnt(3)
+; GFX7-NEXT: v_and_b32_e32 v48, 0x8000, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; GFX7-NEXT: v_and_b32_e32 v31, 0x8000, v31
+; GFX7-NEXT: v_or_b32_e32 v15, v15, v31
; GFX7-NEXT: v_or_b32_e32 v3, v3, v19
; GFX7-NEXT: v_or_b32_e32 v2, v2, v18
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
; GFX7-NEXT: v_or_b32_e32 v0, v0, v16
+; GFX7-NEXT: v_or_b32_e32 v48, v53, v48
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX7-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v35, 0x8000, v31
-; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; GFX7-NEXT: v_and_b32_e32 v31, 0x8000, v31
-; GFX7-NEXT: v_or_b32_e32 v15, v15, v31
-; GFX7-NEXT: v_or_b32_e32 v35, v37, v35
; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GFX7-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GFX7-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX7-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GFX7-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX7-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX7-NEXT: v_or_b32_e32 v0, v33, v0
-; GFX7-NEXT: v_or_b32_e32 v2, v55, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v54, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v53, v4
-; GFX7-NEXT: v_or_b32_e32 v5, v52, v5
-; GFX7-NEXT: v_or_b32_e32 v6, v51, v6
-; GFX7-NEXT: v_or_b32_e32 v7, v50, v7
-; GFX7-NEXT: v_or_b32_e32 v8, v49, v8
-; GFX7-NEXT: v_or_b32_e32 v9, v48, v9
-; GFX7-NEXT: v_or_b32_e32 v10, v39, v10
-; GFX7-NEXT: v_or_b32_e32 v11, v38, v11
-; GFX7-NEXT: v_or_b32_e32 v12, v36, v12
-; GFX7-NEXT: v_or_b32_e32 v13, v34, v13
-; GFX7-NEXT: v_or_b32_e32 v14, v32, v14
-; GFX7-NEXT: v_or_b32_e32 v15, v35, v15
+; GFX7-NEXT: v_or_b32_e32 v0, v38, v0
+; GFX7-NEXT: v_or_b32_e32 v1, v36, v1
+; GFX7-NEXT: v_or_b32_e32 v2, v33, v2
+; GFX7-NEXT: v_or_b32_e32 v3, v49, v3
+; GFX7-NEXT: v_or_b32_e32 v5, v35, v5
+; GFX7-NEXT: v_or_b32_e32 v6, v34, v6
+; GFX7-NEXT: v_or_b32_e32 v7, v32, v7
+; GFX7-NEXT: v_or_b32_e32 v9, v55, v9
+; GFX7-NEXT: v_or_b32_e32 v10, v54, v10
+; GFX7-NEXT: v_or_b32_e32 v11, v51, v11
+; GFX7-NEXT: v_or_b32_e32 v12, v50, v12
+; GFX7-NEXT: v_or_b32_e32 v13, v39, v13
+; GFX7-NEXT: v_or_b32_e32 v14, v37, v14
+; GFX7-NEXT: v_or_b32_e32 v15, v48, v15
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_copysign_v32bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index ba404a8d573ab..13bb2393e1449 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -2101,99 +2101,86 @@ define <32 x half> @v_copysign_v32f32(<32 x half> %mag, <32 x half> %sign) {
; SI-LABEL: v_copysign_v32f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v58, 0x8000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v33, 0x7fff, v14
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_and_b32_e32 v40, 0x8000, v30
+; SI-NEXT: v_or_b32_e32 v37, v33, v40
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v29
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v13
+; SI-NEXT: v_or_b32_e32 v39, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v28
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v12
+; SI-NEXT: v_or_b32_e32 v50, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v27
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v11
+; SI-NEXT: v_or_b32_e32 v51, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v26
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v10
+; SI-NEXT: v_or_b32_e32 v54, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v25
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v9
+; SI-NEXT: v_or_b32_e32 v55, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v24
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v8
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v59, 0x7fff, v1
-; SI-NEXT: v_and_b32_e32 v17, 0x8000, v17
-; SI-NEXT: v_bfe_u32 v1, v1, 16, 15
-; SI-NEXT: v_and_b32_e32 v38, 0x8000, v27
-; SI-NEXT: v_and_b32_e32 v39, 0x7fff, v11
-; SI-NEXT: v_and_b32_e32 v48, 0x8000, v26
-; SI-NEXT: v_and_b32_e32 v49, 0x7fff, v10
-; SI-NEXT: v_and_b32_e32 v50, 0x8000, v25
-; SI-NEXT: v_and_b32_e32 v51, 0x7fff, v9
-; SI-NEXT: v_and_b32_e32 v40, 0x8000, v22
-; SI-NEXT: v_and_b32_e32 v41, 0x7fff, v6
-; SI-NEXT: v_or_b32_e32 v1, v1, v17
-; SI-NEXT: v_and_b32_e32 v52, 0x8000, v24
-; SI-NEXT: v_and_b32_e32 v53, 0x7fff, v8
-; SI-NEXT: v_and_b32_e32 v54, 0x8000, v23
-; SI-NEXT: v_and_b32_e32 v55, 0x7fff, v7
-; SI-NEXT: v_and_b32_e32 v42, 0x8000, v21
-; SI-NEXT: v_and_b32_e32 v43, 0x7fff, v5
-; SI-NEXT: v_and_b32_e32 v44, 0x8000, v20
-; SI-NEXT: v_and_b32_e32 v45, 0x7fff, v4
-; SI-NEXT: v_and_b32_e32 v46, 0x8000, v19
-; SI-NEXT: v_and_b32_e32 v47, 0x7fff, v3
-; SI-NEXT: v_and_b32_e32 v56, 0x8000, v18
-; SI-NEXT: v_and_b32_e32 v57, 0x7fff, v2
-; SI-NEXT: v_or_b32_e32 v38, v39, v38
-; SI-NEXT: v_or_b32_e32 v39, v49, v48
-; SI-NEXT: v_or_b32_e32 v48, v51, v50
-; SI-NEXT: v_or_b32_e32 v51, v41, v40
-; SI-NEXT: v_or_b32_e32 v40, v59, v58
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v49, v53, v52
-; SI-NEXT: v_or_b32_e32 v50, v55, v54
-; SI-NEXT: v_or_b32_e32 v52, v43, v42
-; SI-NEXT: v_or_b32_e32 v53, v45, v44
-; SI-NEXT: v_or_b32_e32 v54, v47, v46
-; SI-NEXT: v_or_b32_e32 v55, v57, v56
-; SI-NEXT: v_or_b32_e32 v1, v40, v1
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v42, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v23
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v7
+; SI-NEXT: v_or_b32_e32 v32, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v22
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v6
+; SI-NEXT: v_or_b32_e32 v34, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v21
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v5
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; SI-NEXT: v_and_b32_e32 v32, 0x8000, v30
-; SI-NEXT: v_and_b32_e32 v33, 0x7fff, v14
-; SI-NEXT: v_and_b32_e32 v34, 0x8000, v29
-; SI-NEXT: v_and_b32_e32 v35, 0x7fff, v13
-; SI-NEXT: v_or_b32_e32 v32, v33, v32
-; SI-NEXT: v_and_b32_e32 v33, 0x8000, v16
-; SI-NEXT: v_or_b32_e32 v34, v35, v34
-; SI-NEXT: v_and_b32_e32 v35, 0x7fff, v0
-; SI-NEXT: v_and_b32_e32 v36, 0x8000, v28
-; SI-NEXT: v_and_b32_e32 v37, 0x7fff, v12
-; SI-NEXT: v_or_b32_e32 v33, v35, v33
+; SI-NEXT: v_or_b32_e32 v35, v40, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v20
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v40, 0x7fff, v4
+; SI-NEXT: v_and_b32_e32 v24, 0x8000, v24
+; SI-NEXT: v_bfe_u32 v8, v8, 16, 15
+; SI-NEXT: v_and_b32_e32 v20, 0x8000, v20
+; SI-NEXT: v_bfe_u32 v4, v4, 16, 15
+; SI-NEXT: v_or_b32_e32 v8, v8, v24
+; SI-NEXT: v_or_b32_e32 v4, v4, v20
+; SI-NEXT: v_and_b32_e32 v36, 0x8000, v17
+; SI-NEXT: v_and_b32_e32 v38, 0x7fff, v1
+; SI-NEXT: v_and_b32_e32 v41, 0x8000, v16
+; SI-NEXT: v_and_b32_e32 v52, 0x7fff, v0
+; SI-NEXT: v_or_b32_e32 v40, v40, v33
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_or_b32_e32 v36, v38, v36
+; SI-NEXT: v_or_b32_e32 v38, v52, v41
+; SI-NEXT: v_or_b32_e32 v4, v40, v4
+; SI-NEXT: v_or_b32_e32 v8, v42, v8
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v49, 0x7fff, v3
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v19
+; SI-NEXT: v_and_b32_e32 v48, 0x7fff, v2
+; SI-NEXT: v_or_b32_e32 v49, v49, v33
+; SI-NEXT: v_and_b32_e32 v33, 0x8000, v18
+; SI-NEXT: v_or_b32_e32 v33, v48, v33
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_or_b32_e32 v36, v37, v36
-; SI-NEXT: v_and_b32_e32 v37, 0x7fff, v15
+; SI-NEXT: v_and_b32_e32 v53, 0x7fff, v15
; SI-NEXT: v_bfe_u32 v15, v15, 16, 15
; SI-NEXT: v_and_b32_e32 v30, 0x8000, v30
; SI-NEXT: v_bfe_u32 v14, v14, 16, 15
@@ -2207,20 +2194,18 @@ define <32 x half> @v_copysign_v32f32(<32 x half> %mag, <32 x half> %sign) {
; SI-NEXT: v_bfe_u32 v10, v10, 16, 15
; SI-NEXT: v_and_b32_e32 v25, 0x8000, v25
; SI-NEXT: v_bfe_u32 v9, v9, 16, 15
-; SI-NEXT: v_and_b32_e32 v24, 0x8000, v24
-; SI-NEXT: v_bfe_u32 v8, v8, 16, 15
; SI-NEXT: v_and_b32_e32 v23, 0x8000, v23
; SI-NEXT: v_bfe_u32 v7, v7, 16, 15
; SI-NEXT: v_and_b32_e32 v22, 0x8000, v22
; SI-NEXT: v_bfe_u32 v6, v6, 16, 15
; SI-NEXT: v_and_b32_e32 v21, 0x8000, v21
; SI-NEXT: v_bfe_u32 v5, v5, 16, 15
-; SI-NEXT: v_and_b32_e32 v20, 0x8000, v20
-; SI-NEXT: v_bfe_u32 v4, v4, 16, 15
; SI-NEXT: v_and_b32_e32 v19, 0x8000, v19
; SI-NEXT: v_bfe_u32 v3, v3, 16, 15
; SI-NEXT: v_and_b32_e32 v18, 0x8000, v18
; SI-NEXT: v_bfe_u32 v2, v2, 16, 15
+; SI-NEXT: v_and_b32_e32 v17, 0x8000, v17
+; SI-NEXT: v_bfe_u32 v1, v1, 16, 15
; SI-NEXT: v_and_b32_e32 v16, 0x8000, v16
; SI-NEXT: v_bfe_u32 v0, v0, 16, 15
; SI-NEXT: v_or_b32_e32 v14, v14, v30
@@ -2229,50 +2214,48 @@ define <32 x half> @v_copysign_v32f32(<32 x half> %mag, <32 x half> %sign) {
; SI-NEXT: v_or_b32_e32 v11, v11, v27
; SI-NEXT: v_or_b32_e32 v10, v10, v26
; SI-NEXT: v_or_b32_e32 v9, v9, v25
-; SI-NEXT: v_or_b32_e32 v8, v8, v24
; SI-NEXT: v_or_b32_e32 v7, v7, v23
; SI-NEXT: v_or_b32_e32 v6, v6, v22
; SI-NEXT: v_or_b32_e32 v5, v5, v21
-; SI-NEXT: v_or_b32_e32 v4, v4, v20
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v48, 0x8000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_and_b32_e32 v31, 0x8000, v31
+; SI-NEXT: v_or_b32_e32 v15, v15, v31
; SI-NEXT: v_or_b32_e32 v3, v3, v19
; SI-NEXT: v_or_b32_e32 v2, v2, v18
+; SI-NEXT: v_or_b32_e32 v1, v1, v17
; SI-NEXT: v_or_b32_e32 v0, v0, v16
+; SI-NEXT: v_or_b32_e32 v48, v53, v48
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v35, 0x8000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: v_and_b32_e32 v31, 0x8000, v31
-; SI-NEXT: v_or_b32_e32 v15, v15, v31
-; SI-NEXT: v_or_b32_e32 v35, v37, v35
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_or_b32_e32 v0, v33, v0
-; SI-NEXT: v_or_b32_e32 v2, v55, v2
-; SI-NEXT: v_or_b32_e32 v3, v54, v3
-; SI-NEXT: v_or_b32_e32 v4, v53, v4
-; SI-NEXT: v_or_b32_e32 v5, v52, v5
-; SI-NEXT: v_or_b32_e32 v6, v51, v6
-; SI-NEXT: v_or_b32_e32 v7, v50, v7
-; SI-NEXT: v_or_b32_e32 v8, v49, v8
-; SI-NEXT: v_or_b32_e32 v9, v48, v9
-; SI-NEXT: v_or_b32_e32 v10, v39, v10
-; SI-NEXT: v_or_b32_e32 v11, v38, v11
-; SI-NEXT: v_or_b32_e32 v12, v36, v12
-; SI-NEXT: v_or_b32_e32 v13, v34, v13
-; SI-NEXT: v_or_b32_e32 v14, v32, v14
-; SI-NEXT: v_or_b32_e32 v15, v35, v15
+; SI-NEXT: v_or_b32_e32 v0, v38, v0
+; SI-NEXT: v_or_b32_e32 v1, v36, v1
+; SI-NEXT: v_or_b32_e32 v2, v33, v2
+; SI-NEXT: v_or_b32_e32 v3, v49, v3
+; SI-NEXT: v_or_b32_e32 v5, v35, v5
+; SI-NEXT: v_or_b32_e32 v6, v34, v6
+; SI-NEXT: v_or_b32_e32 v7, v32, v7
+; SI-NEXT: v_or_b32_e32 v9, v55, v9
+; SI-NEXT: v_or_b32_e32 v10, v54, v10
+; SI-NEXT: v_or_b32_e32 v11, v51, v11
+; SI-NEXT: v_or_b32_e32 v12, v50, v12
+; SI-NEXT: v_or_b32_e32 v13, v39, v13
+; SI-NEXT: v_or_b32_e32 v14, v37, v14
+; SI-NEXT: v_or_b32_e32 v15, v48, v15
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_copysign_v32f32:
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 6101d89f31186..961945b355f22 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -29,61 +29,57 @@ body: |
; CHECK-NEXT: dead undef [[DEF:%[0-9]+]].sub0:vreg_64 = IMPLICIT_DEF
; CHECK-NEXT: dead [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: dead undef [[DEF2:%[0-9]+]].sub0:vreg_64 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX16_IMM renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
- ; CHECK-NEXT: renamable $sgpr24 = IMPLICIT_DEF implicit-def $sgpr25
- ; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+ ; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+ ; CHECK-NEXT: renamable $sgpr40 = IMPLICIT_DEF implicit-def $sgpr41
; CHECK-NEXT: $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.6, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.4, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s256), addrspace 4)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: SI_RETURN
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12 = IMPLICIT_DEF
- ; CHECK-NEXT: SI_SPILL_S512_SAVE renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
- ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr44 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, renamable $sgpr44_sgpr45_sgpr46_sgpr47, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
- ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], undef renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], undef renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: S_BRANCH %bb.7
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_XOR_B64_term $exec, undef renamable $sgpr4_sgpr5, implicit-def $scc
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.8, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: renamable $sgpr41 = COPY undef renamable $sgpr40, implicit-def $sgpr40
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.7, implicit undef $vcc
; CHECK-NEXT: S_BRANCH %bb.6
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index 7dafdf5d0a3d2..56265bdbcd359 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -29,58 +29,55 @@ body: |
; CHECK-NEXT: dead renamable $sgpr5 = IMPLICIT_DEF
; CHECK-NEXT: dead undef [[DEF3:%[0-9]+]].sub1:vreg_64 = IMPLICIT_DEF
; CHECK-NEXT: dead renamable $sgpr5 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
- ; CHECK-NEXT: renamable $sgpr24 = IMPLICIT_DEF implicit-def $sgpr25
; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+ ; CHECK-NEXT: renamable $sgpr40 = IMPLICIT_DEF implicit-def $sgpr41
; CHECK-NEXT: $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.5, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.4
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.3, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s256), addrspace 4)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: SI_RETURN
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
- ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+ ; CHECK-NEXT: renamable $sgpr44 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, renamable $sgpr44_sgpr45_sgpr46_sgpr47, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
- ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], undef renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], undef renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: S_BRANCH %bb.6
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+ ; CHECK-NEXT: liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_XOR_B64_term $exec, undef renamable $sgpr4_sgpr5, implicit-def $scc
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.7, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: renamable $sgpr41 = COPY undef renamable $sgpr40, implicit-def $sgpr40
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.6, implicit undef $vcc
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/issue48473.mir b/llvm/test/CodeGen/AMDGPU/issue48473.mir
index d655338dabae9..ab8f8059ae467 100644
--- a/llvm/test/CodeGen/AMDGPU/issue48473.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue48473.mir
@@ -43,7 +43,7 @@
# %25 to $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
# CHECK-LABEL: name: issue48473
-# CHECK: S_NOP 0, implicit killed renamable $sgpr20_sgpr21_sgpr22_sgpr23, implicit killed renamable $sgpr88_sgpr89_sgpr90_sgpr91, implicit killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, implicit killed renamable $sgpr40_sgpr41_sgpr42_sgpr43, implicit killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, implicit killed renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, implicit killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, implicit killed renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit killed renamable $sgpr16_sgpr17_sgpr18_sgpr19, implicit killed renamable $sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, implicit killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, implicit killed renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr44_sgpr45_sgpr46_sgpr47, implicit killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
+# CHECK: S_NOP 0, implicit killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, implicit killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47, implicit killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, implicit killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, implicit killed renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, implicit killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, implicit killed renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, implicit killed renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, implicit killed renamable $sgpr96_sgpr97_sgpr98_sgpr99, implicit killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
---
name: issue48473
diff --git a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
index afa9cc9af9ca2..b0797a01817de 100644
--- a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
+++ b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
@@ -31,9 +31,9 @@ body: |
; CHECK-LABEL: name: temp_vgpr_to_agpr_should_not_undo_split_with_remat
; CHECK: liveins: $vgpr0, $sgpr4_sgpr5
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr39 = IMPLICIT_DEF
; CHECK-NEXT: dead renamable $vgpr1 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr48 = IMPLICIT_DEF
; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF
; CHECK-NEXT: renamable $vgpr3 = IMPLICIT_DEF
; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF
@@ -57,9 +57,9 @@ body: |
; CHECK-NEXT: renamable $vgpr22 = IMPLICIT_DEF
; CHECK-NEXT: KILL killed renamable $vgpr2, killed renamable $vgpr3, killed renamable $vgpr4, killed renamable $vgpr5, killed renamable $vgpr6, killed renamable $vgpr7, killed renamable $vgpr8, killed renamable $vgpr9, killed renamable $vgpr10, killed renamable $vgpr11, killed renamable $vgpr12, killed renamable $vgpr13, killed renamable $vgpr14, killed renamable $vgpr15, killed renamable $vgpr16
; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38
- ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54
- ; CHECK-NEXT: KILL killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr17, killed renamable $vgpr18, killed renamable $vgpr19, killed renamable $vgpr20, killed renamable $vgpr21, killed renamable $vgpr22
- ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38, implicit killed renamable $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54
+ ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+ ; CHECK-NEXT: KILL killed renamable $vgpr39, killed renamable $vgpr48, killed renamable $vgpr17, killed renamable $vgpr18, killed renamable $vgpr19, killed renamable $vgpr20, killed renamable $vgpr21, killed renamable $vgpr22
+ ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; CHECK-NEXT: S_ENDPGM 0
%1:vgpr_32 = IMPLICIT_DEF
%2:vgpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index 504739de571a9..40dc296ded423 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -7965,7 +7965,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_cmp_eq_f32_e64 s[8:9], 0, v39
; GFX8-NEXT: s_and_b64 vcc, s[6:7], vcc
; GFX8-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v42, v32, v31, vcc
; GFX8-NEXT: s_and_b64 vcc, s[8:9], s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v15
; GFX8-NEXT: v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -7978,9 +7978,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_and_b32_e32 v52, 0xffff0000, v21
; GFX8-NEXT: v_and_b32_e32 v54, 0xffff0000, v20
; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; GFX8-NEXT: v_and_b32_e32 v40, 0xffff0000, v18
-; GFX8-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; GFX8-NEXT: v_and_b32_e32 v42, 0xffff0000, v16
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v35, 16, v53
; GFX8-NEXT: v_and_b32_e32 v36, 0xffff0000, v53
@@ -8158,76 +8155,79 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v2
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
; GFX8-NEXT: v_cndmask_b32_e32 v52, v55, v54, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v52, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v52
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v55, v40
+; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v52, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v52
; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v54
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v31, v55
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v54, v52, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v54, 16, v31
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v54
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v52
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v52, v54, v52, vcc
-; GFX8-NEXT: v_and_b32_e32 v54, 0xffff0000, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; GFX8-NEXT: v_lshrrev_b32_e32 v40, 16, v1
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT: v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v41, v41
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v40, v41
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v55
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v54
-; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v40, 16, v16
-; GFX8-NEXT: v_lshrrev_b32_e32 v41, 16, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v52, v31, v52, vcc
+; GFX8-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v54, 16, v17
+; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v17
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v42, v42
-; GFX8-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT: v_lshlrev_b32_e32 v42, 16, v40
-; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v41, v42
-; GFX8-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v40
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v55
+; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v40, v55
+; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v31
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v15
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v53
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v53, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX8-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX8-NEXT: v_and_b32_e32 v40, 0xffff0000, v16
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT: v_cndmask_b32_e32 v53, v53, v15, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v53
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v15
+; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v41, v40
-; GFX8-NEXT: v_cndmask_b32_e32 v53, v53, v15, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v53
+; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v31
+; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v15
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v53
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v53, vcc
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v53, v15, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v15
+; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v40, v53
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v31
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v53
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v15
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v53, v15, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v14
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
-; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v30
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v14
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX8-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX8-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v30
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v14
-; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v40, v53
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v14
+; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v53, v31
; GFX8-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v30
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v53
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v31
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v14
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8238,8 +8238,8 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
; GFX8-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
-; GFX8-NEXT: v_lshlrev_b32_e32 v53, 16, v13
-; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v53, v30
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v13
+; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v31, v30
; GFX8-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8346,9 +8346,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v6
; GFX8-NEXT: v_cmp_gt_f32_e32 vcc, v24, v23
; GFX8-NEXT: v_cndmask_b32_e32 v22, v22, v6, vcc
-; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v23
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v6
@@ -8472,7 +8469,10 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_or_b32_sdwa v12, v12, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v32
; GFX8-NEXT: v_or_b32_sdwa v13, v13, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v42
+; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX8-NEXT: v_or_b32_sdwa v14, v14, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v33
; GFX8-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -8523,7 +8523,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_cmp_eq_f32_e64 s[8:9], 0, v39
; GFX900-NEXT: s_and_b64 vcc, s[6:7], vcc
; GFX900-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; GFX900-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX900-NEXT: v_cndmask_b32_e32 v42, v32, v31, vcc
; GFX900-NEXT: s_and_b64 vcc, s[8:9], s[4:5]
; GFX900-NEXT: v_lshrrev_b32_e32 v34, 16, v15
; GFX900-NEXT: v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -8536,9 +8536,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_and_b32_e32 v53, 0xffff0000, v21
; GFX900-NEXT: v_and_b32_e32 v54, 0xffff0000, v20
; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; GFX900-NEXT: v_and_b32_e32 v40, 0xffff0000, v18
-; GFX900-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; GFX900-NEXT: v_and_b32_e32 v42, 0xffff0000, v16
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_lshrrev_b32_e32 v35, 16, v52
; GFX900-NEXT: v_and_b32_e32 v36, 0xffff0000, v52
@@ -8716,76 +8713,79 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v2
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
; GFX900-NEXT: v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v53
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v55, v40
+; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v18
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v53
; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v54
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v31, v55
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v54, 16, v31
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v54
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v53
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX900-NEXT: v_and_b32_e32 v54, 0xffff0000, v1
-; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; GFX900-NEXT: v_lshrrev_b32_e32 v40, 16, v1
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT: v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v41, v41
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v40, v41
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v55
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v54
-; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; GFX900-NEXT: v_lshrrev_b32_e32 v40, 16, v16
-; GFX900-NEXT: v_lshrrev_b32_e32 v41, 16, v0
+; GFX900-NEXT: v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX900-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
+; GFX900-NEXT: v_lshrrev_b32_e32 v54, 16, v17
+; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v17
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v42, v42
-; GFX900-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT: v_lshlrev_b32_e32 v42, 16, v40
-; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v41, v42
-; GFX900-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v40
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v55
+; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v40, v55
+; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v31
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v15
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
+; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX900-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX900-NEXT: v_and_b32_e32 v40, 0xffff0000, v16
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v15
+; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v41, v40
-; GFX900-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v52
+; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v31
+; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v15
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v52
+; GFX900-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v15
+; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v40, v52
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v15
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v14
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX900-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v14
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX900-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX900-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v14
-; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v40, v52
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v14
+; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v52, v31
; GFX900-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v31
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v14
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
; GFX900-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8796,8 +8796,8 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
; GFX900-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX900-NEXT: v_lshlrev_b32_e32 v30, 16, v29
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v13
-; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v52, v30
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v13
+; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, v31, v30
; GFX900-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX900-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8882,9 +8882,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_lshlrev_b32_e32 v24, 16, v7
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX900-NEXT: v_lshlrev_b32_e32 v24, 16, v23
-; GFX900-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX900-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX900-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX900-NEXT: v_cndmask_b32_e32 v23, v23, v7, vcc
@@ -9001,8 +8998,12 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v17
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], 0, v0
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
; GFX900-NEXT: s_mov_b32 s4, 0x5040100
+; GFX900-NEXT: v_perm_b32 v14, v42, v14, s4
+; GFX900-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX900-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX900-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX900-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
; GFX900-NEXT: v_perm_b32 v0, v55, v0, s4
; GFX900-NEXT: v_perm_b32 v1, v54, v1, s4
; GFX900-NEXT: v_perm_b32 v2, v53, v2, s4
@@ -9017,7 +9018,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_perm_b32 v11, v35, v11, s4
; GFX900-NEXT: v_perm_b32 v12, v34, v12, s4
; GFX900-NEXT: v_perm_b32 v13, v32, v13, s4
-; GFX900-NEXT: v_perm_b32 v14, v31, v14, s4
; GFX900-NEXT: v_perm_b32 v15, v33, v15, s4
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
@@ -9067,7 +9067,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_cmp_eq_f32_e64 s[2:3], 0, v38
; GFX950-NEXT: v_cmp_eq_f32_e64 s[4:5], 0, v39
; GFX950-NEXT: s_and_b64 vcc, s[2:3], vcc
-; GFX950-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v42, v32, v31, vcc
; GFX950-NEXT: s_and_b64 vcc, s[4:5], s[0:1]
; GFX950-NEXT: v_cndmask_b32_e32 v32, v36, v35, vcc
; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
@@ -9079,9 +9079,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_and_b32_e32 v53, 0xffff0000, v21
; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v20
; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v18
-; GFX950-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; GFX950-NEXT: v_and_b32_e32 v42, 0xffff0000, v16
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_lshrrev_b32_e32 v35, 16, v52
; GFX950-NEXT: v_and_b32_e32 v36, 0xffff0000, v52
@@ -9280,91 +9277,93 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v53
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v18
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v53
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v53
; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v55, v40
-; GFX950-NEXT: v_lshrrev_b32_e32 v40, 16, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v31, v55
+; GFX950-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v54, 16, v31
+; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v54
+; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
+; GFX950-NEXT: v_lshrrev_b32_e32 v54, 16, v17
+; GFX950-NEXT: v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v17
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v31
+; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v31
+; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v40, v55
+; GFX950-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v54
; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; GFX950-NEXT: v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v1
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v54, v54
+; GFX950-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v41, v41
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v54
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v40, v41
-; GFX950-NEXT: v_lshrrev_b32_e32 v41, 16, v0
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v16
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
+; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v31
+; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v31
+; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v55
+; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v41, v40
+; GFX950-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT: v_lshrrev_b32_e32 v40, 16, v16
-; GFX950-NEXT: v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v42, v42
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v55
-; GFX950-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v42, 16, v40
-; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v41, v42
-; GFX950-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v40
-; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v15
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v52
+; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v15
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v52
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v15
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v15
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v15
-; GFX950-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v41, v40
-; GFX950-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v40, v52
+; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v14
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX950-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v14
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v14
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v14
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v14
; GFX950-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v40, v52
-; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v52, v31
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v31
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -9374,11 +9373,11 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc
; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v13
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v13
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], 0, v13
; GFX950-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v52, v30
+; GFX950-NEXT: v_cmp_gt_f32_e32 vcc, v31, v30
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v29
@@ -9634,8 +9633,9 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_perm_b32 v11, v35, v11, s0
; GFX950-NEXT: v_perm_b32 v12, v34, v12, s0
; GFX950-NEXT: v_perm_b32 v13, v32, v13, s0
-; GFX950-NEXT: v_perm_b32 v14, v31, v14, s0
+; GFX950-NEXT: v_perm_b32 v14, v42, v14, s0
; GFX950-NEXT: v_perm_b32 v15, v33, v15, s0
+; GFX950-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_maximumnum_v32bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.ll
index 832c127622751..edaacdffcc8e5 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.ll
@@ -6437,52 +6437,63 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX7-LABEL: v_maximumnum_v32f16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32
; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT: v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT: v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT: v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT: v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT: v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT: v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT: v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT: v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT: v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT: v_max_f32_e32 v31, v32, v31
-; GFX7-NEXT: v_max_f32_e32 v33, v35, v33
-; GFX7-NEXT: v_max_f32_e32 v32, v37, v36
-; GFX7-NEXT: v_max_f32_e32 v35, v39, v38
-; GFX7-NEXT: v_max_f32_e32 v36, v49, v48
-; GFX7-NEXT: v_max_f32_e32 v37, v51, v50
-; GFX7-NEXT: v_max_f32_e32 v39, v53, v52
-; GFX7-NEXT: v_max_f32_e32 v48, v55, v54
+; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT: v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT: v_max_f32_e32 v31, v33, v32
+; GFX7-NEXT: v_max_f32_e32 v33, v36, v35
+; GFX7-NEXT: v_max_f32_e32 v32, v38, v37
+; GFX7-NEXT: v_max_f32_e32 v35, v48, v39
+; GFX7-NEXT: v_max_f32_e32 v36, v50, v49
+; GFX7-NEXT: v_max_f32_e32 v37, v52, v51
+; GFX7-NEXT: v_max_f32_e32 v39, v54, v53
+; GFX7-NEXT: v_max_f32_e32 v48, v40, v55
; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v19
@@ -6504,46 +6515,37 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
; GFX7-NEXT: v_cvt_f32_f16_e32 v17, v17
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
; GFX7-NEXT: v_max_f32_e32 v53, v54, v53
-; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_max_f32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT: v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT: v_max_f32_e32 v49, v41, v40
+; GFX7-NEXT: v_max_f32_e32 v49, v42, v41
; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v34
; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v0
; GFX7-NEXT: v_max_f32_e32 v3, v3, v19
; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v42
; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT: v_max_f32_e32 v40, v44, v43
+; GFX7-NEXT: v_max_f32_e32 v40, v45, v44
; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_max_f32_e32 v38, v42, v38
+; GFX7-NEXT: v_max_f32_e32 v38, v43, v38
; GFX7-NEXT: v_max_f32_e32 v54, v41, v55
; GFX7-NEXT: v_or_b32_e32 v3, v3, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX7-NEXT: v_cvt_f32_f16_e32 v18, v18
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v16, v16
@@ -6647,96 +6649,78 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX8-LABEL: v_maximumnum_v32f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT: v_max_f16_sdwa v38, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v39, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v48, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v49, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v50, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v51, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v40, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v41, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v58, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v59, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v17, v17, v17
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v52, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v53, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v54, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v55, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v42, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v43, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v44, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v45, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v46, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v47, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v56, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v57, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v38, v39, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v39, v49, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v48, v51, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v51, v41, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v40, v59, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v17
-; GFX8-NEXT: v_max_f16_sdwa v49, v53, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v50, v55, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v52, v43, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v53, v45, v44 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v54, v47, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v55, v57, v56 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v40
-; GFX8-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX8-NEXT: v_max_f16_sdwa v32, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX8-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX8-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX8-NEXT: v_max_f16_sdwa v33, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v34, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v35, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v36, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v37, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v32, v33, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v33, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v34, v35, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v35, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v36, v37, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v37, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v15, v15, v15
-; GFX8-NEXT: v_max_f16_sdwa v33, v35, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v40, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v39, v33, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v33, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v40, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v50, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v33, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v40, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v40, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v32, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v34, v15, v15
+; GFX8-NEXT: v_max_f16_sdwa v15, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v51, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v54, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX8-NEXT: v_max_f16_sdwa v55, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v28, v28, v28
+; GFX8-NEXT: v_max_f16_e32 v12, v12, v12
+; GFX8-NEXT: v_max_f16_e32 v24, v24, v24
+; GFX8-NEXT: v_max_f16_e32 v8, v8, v8
+; GFX8-NEXT: v_max_f16_sdwa v38, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v48, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v53, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v41, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v42, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v12, v12, v28
+; GFX8-NEXT: v_max_f16_e32 v8, v8, v24
+; GFX8-NEXT: v_max_f16_sdwa v38, v48, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v48, v41, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v8, v8, v42
+; GFX8-NEXT: v_or_b32_e32 v12, v12, v40
+; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX8-NEXT: v_max_f16_sdwa v15, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v35, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v36, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v52, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v37, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v52, v52, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v49, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v15, v49, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_max_f16_e32 v30, v30, v30
; GFX8-NEXT: v_max_f16_e32 v14, v14, v14
; GFX8-NEXT: v_max_f16_e32 v29, v29, v29
; GFX8-NEXT: v_max_f16_e32 v13, v13, v13
-; GFX8-NEXT: v_max_f16_e32 v28, v28, v28
-; GFX8-NEXT: v_max_f16_e32 v12, v12, v12
; GFX8-NEXT: v_max_f16_e32 v27, v27, v27
; GFX8-NEXT: v_max_f16_e32 v11, v11, v11
; GFX8-NEXT: v_max_f16_e32 v26, v26, v26
; GFX8-NEXT: v_max_f16_e32 v10, v10, v10
; GFX8-NEXT: v_max_f16_e32 v25, v25, v25
; GFX8-NEXT: v_max_f16_e32 v9, v9, v9
-; GFX8-NEXT: v_max_f16_e32 v24, v24, v24
-; GFX8-NEXT: v_max_f16_e32 v8, v8, v8
; GFX8-NEXT: v_max_f16_e32 v23, v23, v23
; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
; GFX8-NEXT: v_max_f16_e32 v22, v22, v22
@@ -6749,42 +6733,43 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
; GFX8-NEXT: v_max_f16_e32 v18, v18, v18
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v17, v17, v17
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: v_max_f16_e32 v16, v16, v16
; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
; GFX8-NEXT: v_max_f16_e32 v14, v14, v30
; GFX8-NEXT: v_max_f16_e32 v13, v13, v29
-; GFX8-NEXT: v_max_f16_e32 v12, v12, v28
; GFX8-NEXT: v_max_f16_e32 v11, v11, v27
; GFX8-NEXT: v_max_f16_e32 v10, v10, v26
; GFX8-NEXT: v_max_f16_e32 v9, v9, v25
-; GFX8-NEXT: v_max_f16_e32 v8, v8, v24
; GFX8-NEXT: v_max_f16_e32 v7, v7, v23
; GFX8-NEXT: v_max_f16_e32 v6, v6, v22
; GFX8-NEXT: v_max_f16_e32 v5, v5, v21
; GFX8-NEXT: v_max_f16_e32 v4, v4, v20
; GFX8-NEXT: v_max_f16_e32 v3, v3, v19
; GFX8-NEXT: v_max_f16_e32 v2, v2, v18
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v17
; GFX8-NEXT: v_max_f16_e32 v0, v0, v16
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v33
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v55
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v54
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v53
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v52
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v51
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v50
-; GFX8-NEXT: v_or_b32_e32 v8, v8, v49
-; GFX8-NEXT: v_or_b32_e32 v9, v9, v48
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v39
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v38
-; GFX8-NEXT: v_or_b32_e32 v12, v12, v36
-; GFX8-NEXT: v_or_b32_e32 v13, v13, v34
-; GFX8-NEXT: v_or_b32_e32 v14, v14, v32
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v35, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v48
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v38
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v15
+; GFX8-NEXT: s_waitcnt vmcnt(3)
+; GFX8-NEXT: v_max_f16_sdwa v49, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v31, v31, v31
-; GFX8-NEXT: v_max_f16_sdwa v35, v37, v35 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v15, v15, v31
-; GFX8-NEXT: v_or_b32_e32 v15, v15, v35
+; GFX8-NEXT: v_max_f16_sdwa v32, v32, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v31, v34, v31
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v33
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v52
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v37
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v36
+; GFX8-NEXT: v_or_b32_e32 v7, v7, v35
+; GFX8-NEXT: v_or_b32_e32 v9, v9, v55
+; GFX8-NEXT: v_or_b32_e32 v10, v10, v54
+; GFX8-NEXT: v_or_b32_e32 v11, v11, v51
+; GFX8-NEXT: v_or_b32_e32 v13, v13, v50
+; GFX8-NEXT: v_or_b32_e32 v14, v14, v39
+; GFX8-NEXT: v_or_b32_e32 v15, v31, v32
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_maximumnum_v32f16:
@@ -10519,52 +10504,63 @@ define <32 x half> @v_maximumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
; GFX7-LABEL: v_maximumnum_v32f16_no_ieee:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32
; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT: v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT: v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT: v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT: v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT: v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT: v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT: v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT: v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT: v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT: v_max_f32_e32 v31, v32, v31
-; GFX7-NEXT: v_max_f32_e32 v33, v35, v33
-; GFX7-NEXT: v_max_f32_e32 v32, v37, v36
-; GFX7-NEXT: v_max_f32_e32 v35, v39, v38
-; GFX7-NEXT: v_max_f32_e32 v36, v49, v48
-; GFX7-NEXT: v_max_f32_e32 v37, v51, v50
-; GFX7-NEXT: v_max_f32_e32 v39, v53, v52
-; GFX7-NEXT: v_max_f32_e32 v48, v55, v54
+; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT: v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT: v_max_f32_e32 v31, v33, v32
+; GFX7-NEXT: v_max_f32_e32 v33, v36, v35
+; GFX7-NEXT: v_max_f32_e32 v32, v38, v37
+; GFX7-NEXT: v_max_f32_e32 v35, v48, v39
+; GFX7-NEXT: v_max_f32_e32 v36, v50, v49
+; GFX7-NEXT: v_max_f32_e32 v37, v52, v51
+; GFX7-NEXT: v_max_f32_e32 v39, v54, v53
+; GFX7-NEXT: v_max_f32_e32 v48, v40, v55
; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v19
@@ -10586,46 +10582,37 @@ define <32 x half> @v_maximumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
; GFX7-NEXT: v_cvt_f32_f16_e32 v17, v17
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
; GFX7-NEXT: v_max_f32_e32 v53, v54, v53
-; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_max_f32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT: v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT: v_max_f32_e32 v49, v41, v40
+; GFX7-NEXT: v_max_f32_e32 v49, v42, v41
; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v34
; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v0
; GFX7-NEXT: v_max_f32_e32 v3, v3, v19
; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v42
; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT: v_max_f32_e32 v40, v44, v43
+; GFX7-NEXT: v_max_f32_e32 v40, v45, v44
; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_max_f32_e32 v38, v42, v38
+; GFX7-NEXT: v_max_f32_e32 v38, v43, v38
; GFX7-NEXT: v_max_f32_e32 v54, v41, v55
; GFX7-NEXT: v_or_b32_e32 v3, v3, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX7-NEXT: v_cvt_f32_f16_e32 v18, v18
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v16, v16
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index d5b2e770283c5..fc200a2ec541f 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -7991,7 +7991,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_cmp_eq_f32_e64 s[8:9], 0, v38
; GFX8-NEXT: s_and_b64 vcc, s[6:7], vcc
; GFX8-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v42, v32, v31, vcc
; GFX8-NEXT: s_and_b64 vcc, s[8:9], s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v15
; GFX8-NEXT: v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -8005,9 +8005,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_and_b32_e32 v53, 0xffff0000, v21
; GFX8-NEXT: v_and_b32_e32 v54, 0xffff0000, v20
; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; GFX8-NEXT: v_and_b32_e32 v40, 0xffff0000, v18
-; GFX8-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; GFX8-NEXT: v_and_b32_e32 v42, 0xffff0000, v16
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v35, 16, v52
; GFX8-NEXT: v_and_b32_e32 v36, 0xffff0000, v52
@@ -8189,76 +8186,79 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v2
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
; GFX8-NEXT: v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v53
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v55, v40
+; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v53
; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v54
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v31, v55
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v54, 16, v31
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v54
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v53
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX8-NEXT: v_and_b32_e32 v54, 0xffff0000, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; GFX8-NEXT: v_lshrrev_b32_e32 v40, 16, v1
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT: v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v41, v41
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v40, v41
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v55
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v54
-; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v40, 16, v16
-; GFX8-NEXT: v_lshrrev_b32_e32 v41, 16, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX8-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v54, 16, v17
+; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX8-NEXT: v_and_b32_e32 v55, 0xffff0000, v17
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v42, v42
-; GFX8-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT: v_lshlrev_b32_e32 v42, 16, v40
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v41, v42
-; GFX8-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v40
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v55
+; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v40, v55
+; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v31
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v15
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX8-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX8-NEXT: v_and_b32_e32 v40, 0xffff0000, v16
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v15
+; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v41, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v41, v40
-; GFX8-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v52
+; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v31
+; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v15
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v52
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v15
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v15
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v14
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v14
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX8-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX8-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX8-NEXT: v_lshlrev_b32_e32 v40, 16, v14
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v14
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v52, v31
; GFX8-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v31
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v14
; GFX8-NEXT: s_and_b64 vcc, vcc, s[4:5]
; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8269,8 +8269,8 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
; GFX8-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
-; GFX8-NEXT: v_lshlrev_b32_e32 v52, 16, v13
-; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v52, v30
+; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v13
+; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v31, v30
; GFX8-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8377,9 +8377,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v6
; GFX8-NEXT: v_cmp_lt_f32_e32 vcc, v24, v23
; GFX8-NEXT: v_cndmask_b32_e32 v22, v22, v6, vcc
-; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v23
; GFX8-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v6
@@ -8503,7 +8500,10 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX8-NEXT: v_or_b32_sdwa v12, v12, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v32
; GFX8-NEXT: v_or_b32_sdwa v13, v13, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v31
+; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v42
+; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX8-NEXT: v_or_b32_sdwa v14, v14, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v33
; GFX8-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -8550,7 +8550,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_cmp_eq_f32_e64 s[8:9], 0, v38
; GFX900-NEXT: s_and_b64 vcc, s[6:7], vcc
; GFX900-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; GFX900-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX900-NEXT: v_cndmask_b32_e32 v42, v32, v31, vcc
; GFX900-NEXT: s_and_b64 vcc, s[8:9], s[4:5]
; GFX900-NEXT: v_lshrrev_b32_e32 v34, 16, v15
; GFX900-NEXT: v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -8564,9 +8564,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_and_b32_e32 v53, 0xffff0000, v21
; GFX900-NEXT: v_and_b32_e32 v54, 0xffff0000, v20
; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; GFX900-NEXT: v_and_b32_e32 v40, 0xffff0000, v18
-; GFX900-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; GFX900-NEXT: v_and_b32_e32 v42, 0xffff0000, v16
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_lshrrev_b32_e32 v35, 16, v52
; GFX900-NEXT: v_and_b32_e32 v36, 0xffff0000, v52
@@ -8748,76 +8745,79 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v2
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
; GFX900-NEXT: v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v53
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v55, v40
+; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v18
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v53
; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v54
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v31, v55
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v54, 16, v31
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v54
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v53
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX900-NEXT: v_and_b32_e32 v54, 0xffff0000, v1
-; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; GFX900-NEXT: v_lshrrev_b32_e32 v40, 16, v1
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT: v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v41, v41
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v40, v41
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v55
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v54
-; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; GFX900-NEXT: v_lshrrev_b32_e32 v40, 16, v16
-; GFX900-NEXT: v_lshrrev_b32_e32 v41, 16, v0
+; GFX900-NEXT: v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX900-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
+; GFX900-NEXT: v_lshrrev_b32_e32 v54, 16, v17
+; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX900-NEXT: v_and_b32_e32 v55, 0xffff0000, v17
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v42, v42
-; GFX900-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT: v_lshlrev_b32_e32 v42, 16, v40
-; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v41, v42
-; GFX900-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v40
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v55
+; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v40, v55
+; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v31
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v15
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
+; GFX900-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX900-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX900-NEXT: v_and_b32_e32 v40, 0xffff0000, v16
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v15
+; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v41, 16, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v41, v40
-; GFX900-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v52
+; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v31
+; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; GFX900-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v15
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v52
+; GFX900-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v15
+; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX900-NEXT: v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v15
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v14
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX900-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v14
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX900-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX900-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT: v_lshlrev_b32_e32 v40, 16, v14
-; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v14
+; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v52, v31
; GFX900-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v31
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v14
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
; GFX900-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8828,8 +8828,8 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
; GFX900-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX900-NEXT: v_lshlrev_b32_e32 v30, 16, v29
-; GFX900-NEXT: v_lshlrev_b32_e32 v52, 16, v13
-; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v52, v30
+; GFX900-NEXT: v_lshlrev_b32_e32 v31, 16, v13
+; GFX900-NEXT: v_cmp_lt_f32_e32 vcc, v31, v30
; GFX900-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX900-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8914,9 +8914,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_lshlrev_b32_e32 v24, 16, v7
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX900-NEXT: v_lshlrev_b32_e32 v24, 16, v23
-; GFX900-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX900-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX900-NEXT: v_cndmask_b32_e32 v7, v7, v23, vcc
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX900-NEXT: v_cndmask_b32_e32 v23, v23, v7, vcc
@@ -9033,8 +9030,12 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, 0, v17
; GFX900-NEXT: v_cmp_eq_u16_e64 s[4:5], s10, v0
; GFX900-NEXT: s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
; GFX900-NEXT: s_mov_b32 s4, 0x5040100
+; GFX900-NEXT: v_perm_b32 v14, v42, v14, s4
+; GFX900-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX900-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX900-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX900-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
; GFX900-NEXT: v_perm_b32 v0, v55, v0, s4
; GFX900-NEXT: v_perm_b32 v1, v54, v1, s4
; GFX900-NEXT: v_perm_b32 v2, v53, v2, s4
@@ -9049,7 +9050,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX900-NEXT: v_perm_b32 v11, v35, v11, s4
; GFX900-NEXT: v_perm_b32 v12, v34, v12, s4
; GFX900-NEXT: v_perm_b32 v13, v32, v13, s4
-; GFX900-NEXT: v_perm_b32 v14, v31, v14, s4
; GFX900-NEXT: v_perm_b32 v15, v33, v15, s4
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
@@ -9094,7 +9094,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_cmp_eq_f32_e64 s[4:5], 0, v38
; GFX950-NEXT: s_and_b64 vcc, s[2:3], vcc
; GFX950-NEXT: v_and_b32_e32 v33, 0xffff0000, v15
-; GFX950-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v42, v32, v31, vcc
; GFX950-NEXT: s_and_b64 vcc, s[4:5], s[0:1]
; GFX950-NEXT: v_lshrrev_b32_e32 v34, 16, v15
; GFX950-NEXT: v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -9108,9 +9108,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_and_b32_e32 v53, 0xffff0000, v21
; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v20
; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v18
-; GFX950-NEXT: v_and_b32_e32 v41, 0xffff0000, v17
-; GFX950-NEXT: v_and_b32_e32 v42, 0xffff0000, v16
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_lshrrev_b32_e32 v35, 16, v52
; GFX950-NEXT: v_and_b32_e32 v36, 0xffff0000, v52
@@ -9313,91 +9310,93 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v53, v53
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v53
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v18
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v53
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v53
; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v55, v40
-; GFX950-NEXT: v_lshrrev_b32_e32 v40, 16, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v31, v55
+; GFX950-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v54, 16, v31
+; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v54
+; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
+; GFX950-NEXT: v_lshrrev_b32_e32 v54, 16, v17
+; GFX950-NEXT: v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v1
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v17
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v31
+; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v31
+; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v40, v55
+; GFX950-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v55, 16, v54
; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v55
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; GFX950-NEXT: v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX950-NEXT: v_and_b32_e32 v54, 0xffff0000, v1
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v54, v54
+; GFX950-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX950-NEXT: v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT: v_and_b32_e32 v31, 0xffff0000, v0
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v41, v41
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v54
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v40, v41
-; GFX950-NEXT: v_lshrrev_b32_e32 v41, 16, v0
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX950-NEXT: v_and_b32_e32 v40, 0xffff0000, v16
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
+; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v31
+; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v31
+; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v55
+; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v41, v40
+; GFX950-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v54, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v55
; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT: v_lshrrev_b32_e32 v40, 16, v16
-; GFX950-NEXT: v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX950-NEXT: v_and_b32_e32 v55, 0xffff0000, v0
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v55, v55
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v42, v42
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v55
-; GFX950-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v42, 16, v40
-; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v41, v42
-; GFX950-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v40
-; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v15
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v52
+; GFX950-NEXT: v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v15
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v52
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v15, v15, v52, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v15
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v15
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v15
-; GFX950-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v41, v40
-; GFX950-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX950-NEXT: v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v14
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX950-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v14
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v14
+; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v14
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v14
; GFX950-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v40, v52
-; GFX950-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v52, v31
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT: v_cmp_eq_f32_e32 vcc, 0, v31
; GFX950-NEXT: s_and_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -9407,11 +9406,11 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e32 v13, v13, v29, vcc
; GFX950-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v13
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v13
; GFX950-NEXT: v_cmp_eq_u16_e64 s[0:1], s6, v13
; GFX950-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v52, v30
+; GFX950-NEXT: v_cmp_lt_f32_e32 vcc, v31, v30
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v29, v29, v13, vcc
; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v29
@@ -9667,8 +9666,9 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX950-NEXT: v_perm_b32 v11, v35, v11, s0
; GFX950-NEXT: v_perm_b32 v12, v34, v12, s0
; GFX950-NEXT: v_perm_b32 v13, v32, v13, s0
-; GFX950-NEXT: v_perm_b32 v14, v31, v14, s0
+; GFX950-NEXT: v_perm_b32 v14, v42, v14, s0
; GFX950-NEXT: v_perm_b32 v15, v33, v15, s0
+; GFX950-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_minimumnum_v32bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.ll
index cb80101095780..83ba18e515433 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.ll
@@ -6218,52 +6218,63 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX7-LABEL: v_minimumnum_v32f16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32
; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT: v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT: v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT: v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT: v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT: v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT: v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT: v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT: v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT: v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT: v_min_f32_e32 v31, v32, v31
-; GFX7-NEXT: v_min_f32_e32 v33, v35, v33
-; GFX7-NEXT: v_min_f32_e32 v32, v37, v36
-; GFX7-NEXT: v_min_f32_e32 v35, v39, v38
-; GFX7-NEXT: v_min_f32_e32 v36, v49, v48
-; GFX7-NEXT: v_min_f32_e32 v37, v51, v50
-; GFX7-NEXT: v_min_f32_e32 v39, v53, v52
-; GFX7-NEXT: v_min_f32_e32 v48, v55, v54
+; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT: v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT: v_min_f32_e32 v31, v33, v32
+; GFX7-NEXT: v_min_f32_e32 v33, v36, v35
+; GFX7-NEXT: v_min_f32_e32 v32, v38, v37
+; GFX7-NEXT: v_min_f32_e32 v35, v48, v39
+; GFX7-NEXT: v_min_f32_e32 v36, v50, v49
+; GFX7-NEXT: v_min_f32_e32 v37, v52, v51
+; GFX7-NEXT: v_min_f32_e32 v39, v54, v53
+; GFX7-NEXT: v_min_f32_e32 v48, v40, v55
; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v19
@@ -6285,46 +6296,37 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
; GFX7-NEXT: v_cvt_f32_f16_e32 v17, v17
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
; GFX7-NEXT: v_min_f32_e32 v53, v54, v53
-; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_min_f32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT: v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT: v_min_f32_e32 v49, v41, v40
+; GFX7-NEXT: v_min_f32_e32 v49, v42, v41
; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v34
; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v0
; GFX7-NEXT: v_min_f32_e32 v3, v3, v19
; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v42
; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT: v_min_f32_e32 v40, v44, v43
+; GFX7-NEXT: v_min_f32_e32 v40, v45, v44
; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_min_f32_e32 v38, v42, v38
+; GFX7-NEXT: v_min_f32_e32 v38, v43, v38
; GFX7-NEXT: v_min_f32_e32 v54, v41, v55
; GFX7-NEXT: v_or_b32_e32 v3, v3, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX7-NEXT: v_cvt_f32_f16_e32 v18, v18
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v16, v16
@@ -6428,96 +6430,78 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX8-LABEL: v_minimumnum_v32f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX8-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT: v_max_f16_sdwa v38, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v39, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v48, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v49, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v50, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v51, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v40, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v41, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v58, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v59, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v17, v17, v17
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v52, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v53, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v54, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v55, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v42, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v43, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v44, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v45, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v46, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v47, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v56, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v57, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_sdwa v38, v39, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v39, v49, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v48, v51, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v51, v41, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v40, v59, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v1, v1, v17
-; GFX8-NEXT: v_min_f16_sdwa v49, v53, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v50, v55, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v52, v43, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v53, v45, v44 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v54, v47, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_sdwa v55, v57, v56 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v40
-; GFX8-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX8-NEXT: v_max_f16_sdwa v32, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX8-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX8-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX8-NEXT: v_max_f16_sdwa v33, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v34, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v35, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v36, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v37, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_sdwa v32, v33, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v33, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_sdwa v34, v35, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v35, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_sdwa v36, v37, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_sdwa v37, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v15, v15, v15
-; GFX8-NEXT: v_min_f16_sdwa v33, v35, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v40, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v39, v33, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v33, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v40, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v50, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v33, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v40, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v40, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v32, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v34, v15, v15
+; GFX8-NEXT: v_max_f16_sdwa v15, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v51, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v54, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX8-NEXT: v_min_f16_sdwa v55, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v28, v28, v28
+; GFX8-NEXT: v_max_f16_e32 v12, v12, v12
+; GFX8-NEXT: v_max_f16_e32 v24, v24, v24
+; GFX8-NEXT: v_max_f16_e32 v8, v8, v8
+; GFX8-NEXT: v_max_f16_sdwa v38, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v48, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v53, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v41, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v42, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v12, v12, v28
+; GFX8-NEXT: v_min_f16_e32 v8, v8, v24
+; GFX8-NEXT: v_min_f16_sdwa v38, v48, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_sdwa v48, v41, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v8, v8, v42
+; GFX8-NEXT: v_or_b32_e32 v12, v12, v40
+; GFX8-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX8-NEXT: v_max_f16_sdwa v15, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v35, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v36, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v52, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v37, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v52, v52, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v33, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_sdwa v49, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v33, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_sdwa v15, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_min_f16_sdwa v15, v49, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_max_f16_e32 v30, v30, v30
; GFX8-NEXT: v_max_f16_e32 v14, v14, v14
; GFX8-NEXT: v_max_f16_e32 v29, v29, v29
; GFX8-NEXT: v_max_f16_e32 v13, v13, v13
-; GFX8-NEXT: v_max_f16_e32 v28, v28, v28
-; GFX8-NEXT: v_max_f16_e32 v12, v12, v12
; GFX8-NEXT: v_max_f16_e32 v27, v27, v27
; GFX8-NEXT: v_max_f16_e32 v11, v11, v11
; GFX8-NEXT: v_max_f16_e32 v26, v26, v26
; GFX8-NEXT: v_max_f16_e32 v10, v10, v10
; GFX8-NEXT: v_max_f16_e32 v25, v25, v25
; GFX8-NEXT: v_max_f16_e32 v9, v9, v9
-; GFX8-NEXT: v_max_f16_e32 v24, v24, v24
-; GFX8-NEXT: v_max_f16_e32 v8, v8, v8
; GFX8-NEXT: v_max_f16_e32 v23, v23, v23
; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
; GFX8-NEXT: v_max_f16_e32 v22, v22, v22
@@ -6530,42 +6514,43 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
; GFX8-NEXT: v_max_f16_e32 v18, v18, v18
; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v17, v17, v17
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: v_max_f16_e32 v16, v16, v16
; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
; GFX8-NEXT: v_min_f16_e32 v14, v14, v30
; GFX8-NEXT: v_min_f16_e32 v13, v13, v29
-; GFX8-NEXT: v_min_f16_e32 v12, v12, v28
; GFX8-NEXT: v_min_f16_e32 v11, v11, v27
; GFX8-NEXT: v_min_f16_e32 v10, v10, v26
; GFX8-NEXT: v_min_f16_e32 v9, v9, v25
-; GFX8-NEXT: v_min_f16_e32 v8, v8, v24
; GFX8-NEXT: v_min_f16_e32 v7, v7, v23
; GFX8-NEXT: v_min_f16_e32 v6, v6, v22
; GFX8-NEXT: v_min_f16_e32 v5, v5, v21
; GFX8-NEXT: v_min_f16_e32 v4, v4, v20
; GFX8-NEXT: v_min_f16_e32 v3, v3, v19
; GFX8-NEXT: v_min_f16_e32 v2, v2, v18
+; GFX8-NEXT: v_min_f16_e32 v1, v1, v17
; GFX8-NEXT: v_min_f16_e32 v0, v0, v16
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v33
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v55
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v54
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v53
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v52
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v51
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v50
-; GFX8-NEXT: v_or_b32_e32 v8, v8, v49
-; GFX8-NEXT: v_or_b32_e32 v9, v9, v48
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v39
-; GFX8-NEXT: v_or_b32_e32 v11, v11, v38
-; GFX8-NEXT: v_or_b32_e32 v12, v12, v36
-; GFX8-NEXT: v_or_b32_e32 v13, v13, v34
-; GFX8-NEXT: v_or_b32_e32 v14, v14, v32
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v35, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v48
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v38
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v15
+; GFX8-NEXT: s_waitcnt vmcnt(3)
+; GFX8-NEXT: v_max_f16_sdwa v49, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_max_f16_e32 v31, v31, v31
-; GFX8-NEXT: v_min_f16_sdwa v35, v37, v35 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v15, v15, v31
-; GFX8-NEXT: v_or_b32_e32 v15, v15, v35
+; GFX8-NEXT: v_min_f16_sdwa v32, v32, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v31, v34, v31
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v33
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v52
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v37
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v36
+; GFX8-NEXT: v_or_b32_e32 v7, v7, v35
+; GFX8-NEXT: v_or_b32_e32 v9, v9, v55
+; GFX8-NEXT: v_or_b32_e32 v10, v10, v54
+; GFX8-NEXT: v_or_b32_e32 v11, v11, v51
+; GFX8-NEXT: v_or_b32_e32 v13, v13, v50
+; GFX8-NEXT: v_or_b32_e32 v14, v14, v39
+; GFX8-NEXT: v_or_b32_e32 v15, v31, v32
+; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_minimumnum_v32f16:
@@ -10300,52 +10285,63 @@ define <32 x half> @v_minimumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
; GFX7-LABEL: v_minimumnum_v32f16_no_ieee:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32
; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT: v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT: v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT: v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT: v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT: v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT: v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT: v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT: v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT: v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT: v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT: v_min_f32_e32 v31, v32, v31
-; GFX7-NEXT: v_min_f32_e32 v33, v35, v33
-; GFX7-NEXT: v_min_f32_e32 v32, v37, v36
-; GFX7-NEXT: v_min_f32_e32 v35, v39, v38
-; GFX7-NEXT: v_min_f32_e32 v36, v49, v48
-; GFX7-NEXT: v_min_f32_e32 v37, v51, v50
-; GFX7-NEXT: v_min_f32_e32 v39, v53, v52
-; GFX7-NEXT: v_min_f32_e32 v48, v55, v54
+; GFX7-NEXT: v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT: v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT: v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT: v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT: v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT: v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT: v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT: v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT: v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT: v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT: v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT: v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT: v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT: v_min_f32_e32 v31, v33, v32
+; GFX7-NEXT: v_min_f32_e32 v33, v36, v35
+; GFX7-NEXT: v_min_f32_e32 v32, v38, v37
+; GFX7-NEXT: v_min_f32_e32 v35, v48, v39
+; GFX7-NEXT: v_min_f32_e32 v36, v50, v49
+; GFX7-NEXT: v_min_f32_e32 v37, v52, v51
+; GFX7-NEXT: v_min_f32_e32 v39, v54, v53
+; GFX7-NEXT: v_min_f32_e32 v48, v40, v55
; GFX7-NEXT: v_lshrrev_b32_e32 v50, 16, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v51, 16, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v52, 16, v19
@@ -10367,46 +10363,37 @@ define <32 x half> @v_minimumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
; GFX7-NEXT: v_cvt_f32_f16_e32 v54, v54
; GFX7-NEXT: v_cvt_f32_f16_e32 v17, v17
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
; GFX7-NEXT: v_min_f32_e32 v53, v54, v53
-; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_min_f32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT: v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT: v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT: v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT: v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT: v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT: v_min_f32_e32 v49, v41, v40
+; GFX7-NEXT: v_min_f32_e32 v49, v42, v41
; GFX7-NEXT: v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v38, 16, v34
; GFX7-NEXT: v_lshrrev_b32_e32 v41, 16, v0
; GFX7-NEXT: v_min_f32_e32 v3, v3, v19
; GFX7-NEXT: v_or_b32_e32 v1, v1, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT: v_cvt_f32_f16_e32 v42, v42
; GFX7-NEXT: v_cvt_f32_f16_e32 v38, v38
; GFX7-NEXT: v_cvt_f32_f16_e32 v55, v55
; GFX7-NEXT: v_cvt_f32_f16_e32 v41, v41
; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT: v_min_f32_e32 v40, v44, v43
+; GFX7-NEXT: v_min_f32_e32 v40, v45, v44
; GFX7-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_min_f32_e32 v38, v42, v38
+; GFX7-NEXT: v_min_f32_e32 v38, v43, v38
; GFX7-NEXT: v_min_f32_e32 v54, v41, v55
; GFX7-NEXT: v_or_b32_e32 v3, v3, v17
; GFX7-NEXT: v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX7-NEXT: v_cvt_f32_f16_e32 v18, v18
; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX7-NEXT: v_cvt_f32_f16_e32 v16, v16
diff --git a/llvm/test/CodeGen/AMDGPU/select.f16.ll b/llvm/test/CodeGen/AMDGPU/select.f16.ll
index f698692fc7497..b0531276b5b5c 100644
--- a/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -1264,99 +1264,99 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:64
; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:56
; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:48
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:40
-; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:32
-; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:24
-; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:16
-; VI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:8
+; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:40
+; VI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:32
+; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:24
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:16
+; VI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:116
-; VI-NEXT: v_lshrrev_b32_e32 v42, 16, v14
-; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v30
-; VI-NEXT: v_lshrrev_b32_e32 v44, 16, v13
-; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v29
-; VI-NEXT: v_lshrrev_b32_e32 v46, 16, v12
-; VI-NEXT: v_lshrrev_b32_e32 v47, 16, v28
-; VI-NEXT: v_lshrrev_b32_e32 v56, 16, v11
-; VI-NEXT: v_lshrrev_b32_e32 v57, 16, v27
-; VI-NEXT: v_lshrrev_b32_e32 v58, 16, v10
-; VI-NEXT: v_lshrrev_b32_e32 v59, 16, v26
+; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:116
+; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v14
+; VI-NEXT: v_lshrrev_b32_e32 v52, 16, v30
+; VI-NEXT: v_lshrrev_b32_e32 v44, 16, v29
+; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v27
+; VI-NEXT: v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v26
+; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v9
+; VI-NEXT: v_lshrrev_b32_e32 v47, 16, v25
+; VI-NEXT: v_lshrrev_b32_e32 v57, 16, v8
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v31
-; VI-NEXT: v_cndmask_b32_e32 v42, v43, v42, vcc
+; VI-NEXT: v_cndmask_b32_e32 v59, v52, v51, vcc
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v32
-; VI-NEXT: v_cndmask_b32_e32 v44, v45, v44, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v13
+; VI-NEXT: v_cndmask_b32_e32 v44, v44, v31, vcc
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v33
-; VI-NEXT: v_cndmask_b32_e32 v46, v47, v46, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; VI-NEXT: v_cndmask_b32_e32 v32, v32, v31, vcc
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v34
-; VI-NEXT: v_cndmask_b32_e32 v34, v57, v56, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; VI-NEXT: v_cndmask_b32_e32 v31, v43, v31, vcc
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v35
-; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v9
-; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v25
-; VI-NEXT: v_cndmask_b32_e32 v35, v59, v58, vcc
+; VI-NEXT: v_cndmask_b32_e32 v34, v55, v41, vcc
; VI-NEXT: s_waitcnt vmcnt(13)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v36
-; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v8
-; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v24
-; VI-NEXT: v_cndmask_b32_e32 v31, v43, v31, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v24
+; VI-NEXT: v_cndmask_b32_e32 v35, v47, v53, vcc
; VI-NEXT: s_waitcnt vmcnt(12)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v37
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v15
-; VI-NEXT: v_cndmask_b32_e32 v32, v45, v32, vcc
+; VI-NEXT: v_cndmask_b32_e32 v36, v51, v57, vcc
; VI-NEXT: s_waitcnt vmcnt(11)
-; VI-NEXT: v_lshrrev_b32_e32 v36, 16, v38
+; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v38
; VI-NEXT: s_waitcnt vmcnt(10)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v39
-; VI-NEXT: v_cndmask_b32_e32 v33, v36, v33, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v36, 16, v7
-; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v23
+; VI-NEXT: v_cndmask_b32_e32 v33, v37, v33, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v7
+; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v23
; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v48
-; VI-NEXT: v_cndmask_b32_e32 v36, v37, v36, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v6
-; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v22
+; VI-NEXT: v_cndmask_b32_e32 v37, v39, v37, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v6
+; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v22
; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v49
-; VI-NEXT: v_cndmask_b32_e32 v37, v39, v37, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v5
-; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v21
+; VI-NEXT: v_cndmask_b32_e32 v39, v48, v39, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v5
+; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v21
; VI-NEXT: s_waitcnt vmcnt(7)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v50
-; VI-NEXT: v_cndmask_b32_e32 v39, v48, v39, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v4
-; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v20
-; VI-NEXT: s_waitcnt vmcnt(6)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v51
; VI-NEXT: v_cndmask_b32_e32 v48, v49, v48, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v3
-; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v19
-; VI-NEXT: s_waitcnt vmcnt(5)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v52
+; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v4
+; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v20
+; VI-NEXT: s_waitcnt vmcnt(6)
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v56
; VI-NEXT: v_cndmask_b32_e32 v49, v50, v49, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v2
-; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v18
-; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:108
+; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v3
+; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v19
+; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v58
+; VI-NEXT: v_cndmask_b32_e32 v50, v51, v50, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v2
+; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v18
+; VI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:108
; VI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:92
; VI-NEXT: s_waitcnt vmcnt(6)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v53
-; VI-NEXT: v_cndmask_b32_e32 v50, v51, v50, vcc
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:100
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v45
+; VI-NEXT: v_cndmask_b32_e32 v51, v53, v51, vcc
+; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:100
+; VI-NEXT: v_lshrrev_b32_e32 v41, 16, v1
+; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
; VI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:76
; VI-NEXT: s_waitcnt vmcnt(8)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v54
-; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:68
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v46
+; VI-NEXT: v_cndmask_b32_e32 v41, v45, v41, vcc
+; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:68
; VI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:60
-; VI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:52
-; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v1
-; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
-; VI-NEXT: v_cndmask_b32_e32 v53, v45, v53, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v0
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:52
+; VI-NEXT: v_lshrrev_b32_e32 v52, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v58, 16, v16
; VI-NEXT: s_waitcnt vmcnt(10)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v55
-; VI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:44
-; VI-NEXT: v_cndmask_b32_e32 v45, v58, v45, vcc
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v42
+; VI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44
+; VI-NEXT: v_cndmask_b32_e32 v52, v58, v52, vcc
; VI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:36
; VI-NEXT: s_waitcnt vmcnt(11)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v40
@@ -1364,15 +1364,15 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; VI-NEXT: v_cndmask_b32_e32 v15, v38, v15, vcc
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:20
; VI-NEXT: s_waitcnt vmcnt(12)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v41
-; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:12
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v54
+; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:12
; VI-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc
; VI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:4
; VI-NEXT: s_waitcnt vmcnt(13)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v52
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v55
; VI-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc
; VI-NEXT: s_waitcnt vmcnt(11)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v51
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v53
; VI-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v43
; VI-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc
@@ -1383,16 +1383,16 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v56
; VI-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc
; VI-NEXT: s_waitcnt vmcnt(8)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v54
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v45
; VI-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(7)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v57
; VI-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc
; VI-NEXT: s_waitcnt vmcnt(6)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v59
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v46
; VI-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc
; VI-NEXT: s_waitcnt vmcnt(5)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v55
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v42
; VI-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc
; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v58
@@ -1404,40 +1404,40 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v38
; VI-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v41
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v54
; VI-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v30
; VI-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v45
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v52
; VI-NEXT: v_or_b32_sdwa v0, v0, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v53
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v41
; VI-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v50
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v51
; VI-NEXT: v_or_b32_sdwa v2, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v49
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v50
; VI-NEXT: v_or_b32_sdwa v3, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v48
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v49
; VI-NEXT: v_or_b32_sdwa v4, v4, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v39
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v48
; VI-NEXT: v_or_b32_sdwa v5, v5, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v37
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v39
; VI-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v36
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v37
; VI-NEXT: v_or_b32_sdwa v7, v7, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v32
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v36
; VI-NEXT: v_or_b32_sdwa v8, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; VI-NEXT: v_or_b32_sdwa v9, v9, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v35
-; VI-NEXT: v_or_b32_sdwa v10, v10, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT: v_or_b32_sdwa v9, v9, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v34
+; VI-NEXT: v_or_b32_sdwa v10, v10, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v31
; VI-NEXT: v_or_b32_sdwa v11, v11, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v46
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v32
; VI-NEXT: v_or_b32_sdwa v12, v12, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v44
; VI-NEXT: v_or_b32_sdwa v13, v13, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v42
+; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v59
; VI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll b/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll
index d50a149bb0407..86cd7f2d0f1a4 100644
--- a/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll
+++ b/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll
@@ -48,7 +48,6 @@ define fastcc void @Perl_sv_setnv(i8 %c, ptr %.str.54.3682) nounwind {
; DEFAULT-COST: # %bb.0: # %entry
; DEFAULT-COST-NEXT: addi sp, sp, -32
; DEFAULT-COST-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
-; DEFAULT-COST-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
; DEFAULT-COST-NEXT: li a2, 2
; DEFAULT-COST-NEXT: zext.b a0, a0
; DEFAULT-COST-NEXT: blt a2, a0, .LBB0_3
@@ -64,7 +63,6 @@ define fastcc void @Perl_sv_setnv(i8 %c, ptr %.str.54.3682) nounwind {
; DEFAULT-COST-NEXT: bne a0, a2, .LBB0_5
; DEFAULT-COST-NEXT: .LBB0_4: # %sw.bb3
; DEFAULT-COST-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
-; DEFAULT-COST-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
; DEFAULT-COST-NEXT: addi sp, sp, 32
; DEFAULT-COST-NEXT: ret
; DEFAULT-COST-NEXT: .LBB0_5: # %entry
@@ -74,14 +72,16 @@ define fastcc void @Perl_sv_setnv(i8 %c, ptr %.str.54.3682) nounwind {
; DEFAULT-COST-NEXT: .LBB0_6: # %sw.bb34.i
; DEFAULT-COST-NEXT: sd zero, 8(sp) # 8-byte Folded Spill
; DEFAULT-COST-NEXT: .LBB0_7: # %Perl_sv_reftype.exit
-; DEFAULT-COST-NEXT: li s0, 0
-; DEFAULT-COST-NEXT: li a0, 0
-; DEFAULT-COST-NEXT: li a1, 0
-; DEFAULT-COST-NEXT: jalr s0
-; DEFAULT-COST-NEXT: li a0, 0
+; DEFAULT-COST-NEXT: sd zero, 16(sp) # 8-byte Folded Spill
+; DEFAULT-COST-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT: ld a1, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT: ld a2, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT: jalr a2
+; DEFAULT-COST-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
; DEFAULT-COST-NEXT: ld a1, 8(sp) # 8-byte Folded Reload
-; DEFAULT-COST-NEXT: li a2, 0
-; DEFAULT-COST-NEXT: jalr s0
+; DEFAULT-COST-NEXT: ld a2, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT: ld a3, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT: jalr a3
entry:
switch i8 %c, label %Perl_sv_reftype.exit [
i8 1, label %sw.bb4
diff --git a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
index c46ce8801702b..6eb2227cebab9 100644
--- a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
+++ b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
@@ -19,41 +19,37 @@ define void @caller(ptr %p) personality ptr @__gxx_personality_v0 {
; RV32I-NEXT: .cfi_def_cfa_offset 16
; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
; RV32I-NEXT: .cfi_offset ra, -4
; RV32I-NEXT: .cfi_offset s0, -8
-; RV32I-NEXT: .cfi_offset s1, -12
; RV32I-NEXT: .cfi_remember_state
; RV32I-NEXT: mv s0, a0
; RV32I-NEXT: beqz a0, .LBB0_2
; RV32I-NEXT: # %bb.1: # %bb2
-; RV32I-NEXT: .Ltmp0:
+; RV32I-NEXT: .Ltmp0: # EH_LABEL
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call bar
-; RV32I-NEXT: .Ltmp1:
+; RV32I-NEXT: .Ltmp1: # EH_LABEL
; RV32I-NEXT: j .LBB0_3
; RV32I-NEXT: .LBB0_2: # %bb1
-; RV32I-NEXT: .Ltmp2:
+; RV32I-NEXT: .Ltmp2: # EH_LABEL
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call foo
-; RV32I-NEXT: .Ltmp3:
+; RV32I-NEXT: .Ltmp3: # EH_LABEL
; RV32I-NEXT: .LBB0_3: # %end2
; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
; RV32I-NEXT: .cfi_restore ra
; RV32I-NEXT: .cfi_restore s0
-; RV32I-NEXT: .cfi_restore s1
; RV32I-NEXT: addi sp, sp, 16
; RV32I-NEXT: .cfi_def_cfa_offset 0
; RV32I-NEXT: ret
; RV32I-NEXT: .LBB0_4: # %lpad
; RV32I-NEXT: .cfi_restore_state
-; RV32I-NEXT: .Ltmp4:
-; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: .Ltmp4: # EH_LABEL
+; RV32I-NEXT: sw a0, 4(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call callee
-; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: lw a0, 4(sp) # 4-byte Folded Reload
; RV32I-NEXT: call _Unwind_Resume
;
; RV64I-LABEL: caller:
@@ -62,41 +58,37 @@ define void @caller(ptr %p) personality ptr @__gxx_personality_v0 {
; RV64I-NEXT: .cfi_def_cfa_offset 32
; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: .cfi_offset ra, -8
; RV64I-NEXT: .cfi_offset s0, -16
-; RV64I-NEXT: .cfi_offset s1, -24
; RV64I-NEXT: .cfi_remember_state
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: beqz a0, .LBB0_2
; RV64I-NEXT: # %bb.1: # %bb2
-; RV64I-NEXT: .Ltmp0:
+; RV64I-NEXT: .Ltmp0: # EH_LABEL
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call bar
-; RV64I-NEXT: .Ltmp1:
+; RV64I-NEXT: .Ltmp1: # EH_LABEL
; RV64I-NEXT: j .LBB0_3
; RV64I-NEXT: .LBB0_2: # %bb1
-; RV64I-NEXT: .Ltmp2:
+; RV64I-NEXT: .Ltmp2: # EH_LABEL
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call foo
-; RV64I-NEXT: .Ltmp3:
+; RV64I-NEXT: .Ltmp3: # EH_LABEL
; RV64I-NEXT: .LBB0_3: # %end2
; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
; RV64I-NEXT: .cfi_restore ra
; RV64I-NEXT: .cfi_restore s0
-; RV64I-NEXT: .cfi_restore s1
; RV64I-NEXT: addi sp, sp, 32
; RV64I-NEXT: .cfi_def_cfa_offset 0
; RV64I-NEXT: ret
; RV64I-NEXT: .LBB0_4: # %lpad
; RV64I-NEXT: .cfi_restore_state
-; RV64I-NEXT: .Ltmp4:
-; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: .Ltmp4: # EH_LABEL
+; RV64I-NEXT: sd a0, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call callee
-; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: ld a0, 8(sp) # 8-byte Folded Reload
; RV64I-NEXT: call _Unwind_Resume
entry:
%0 = icmp eq ptr %p, null
diff --git a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
index e264e74fee9d3..3860a9a6818a1 100644
--- a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
+++ b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
@@ -22,17 +22,14 @@
define ptr @ubyte_divmod(ptr %a, ptr %b) {
; CHECK-LABEL: ubyte_divmod:
; CHECK: ## %bb.0: ## %entry
-; CHECK-NEXT: pushq %rbp
-; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: subq $32, %rsp
+; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: subq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 64
-; CHECK-NEXT: .cfi_offset %rbx, -32
-; CHECK-NEXT: .cfi_offset %r14, -24
-; CHECK-NEXT: .cfi_offset %rbp, -16
+; CHECK-NEXT: .cfi_offset %rbx, -24
+; CHECK-NEXT: .cfi_offset %r14, -16
; CHECK-NEXT: movq %rsi, %rbx
; CHECK-NEXT: movq %rdi, %r14
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rsi
@@ -107,7 +104,8 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
; CHECK-NEXT: movzbl %sil, %eax
; CHECK-NEXT: divb %dl
-; CHECK-NEXT: movzbl %ah, %ebp
+; CHECK-NEXT: movzbl %ah, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
; CHECK-NEXT: jmp LBB0_18
; CHECK-NEXT: LBB0_14: ## %cond_true.i200
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
@@ -117,7 +115,7 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
; CHECK-NEXT: movl $4, %edi
; CHECK-NEXT: callq _feraiseexcept
; CHECK-NEXT: LBB0_17: ## %ubyte_ctype_remainder.exit
-; CHECK-NEXT: xorl %ebp, %ebp
+; CHECK-NEXT: movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
; CHECK-NEXT: LBB0_18: ## %ubyte_ctype_remainder.exit
; CHECK-NEXT: movq (%r14), %rax
; CHECK-NEXT: callq *224(%rax)
@@ -168,7 +166,8 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
; CHECK-NEXT: testq %rax, %rax
; CHECK-NEXT: je LBB0_25
; CHECK-NEXT: ## %bb.24: ## %cond_next182
-; CHECK-NEXT: movb %bpl, 16(%rax)
+; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT: movb %cl, 16(%rax)
; CHECK-NEXT: movq %rax, 32(%rbx)
; CHECK-NEXT: movq %rbx, %rax
; CHECK-NEXT: jmp LBB0_28
@@ -182,10 +181,9 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
; CHECK-NEXT: LBB0_27: ## %UnifiedReturnBlock
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: LBB0_28: ## %UnifiedReturnBlock
-; CHECK-NEXT: addq $32, %rsp
+; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r14
-; CHECK-NEXT: popq %rbp
; CHECK-NEXT: retq
entry:
%arg1 = alloca i8, align 1 ; <ptr> [#uses=3]
diff --git a/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll b/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
index 5901b9b8300fa..4d53a6d7873fa 100644
--- a/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
+++ b/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
@@ -12,15 +12,16 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
; CHECK-NEXT: pushl %ebx
; CHECK-NEXT: pushl %edi
; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: pushl %eax
-; CHECK-NEXT: movb $1, %cl
-; CHECK-NEXT: movl $1, %esi
+; CHECK-NEXT: subl $8, %esp
+; CHECK-NEXT: movb $1, %bl
+; CHECK-NEXT: movl $1, %ecx
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_1: # %bb.i5
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: addl %esi, %esi
-; CHECK-NEXT: xorl %ecx, %ecx
+; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: addl %ecx, %ecx
+; CHECK-NEXT: xorl %ebx, %ebx
; CHECK-NEXT: testb $1, %al
; CHECK-NEXT: jne .LBB0_1
; CHECK-NEXT: # %bb.2: # %mp_unexp_mp2d.exit.i
@@ -29,35 +30,34 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
; CHECK-NEXT: jne .LBB0_3
; CHECK-NEXT: # %bb.6: # %cond_next36.i
; CHECK-NEXT: movl $0, 0
-; CHECK-NEXT: movzbl %al, %ebp
-; CHECK-NEXT: andl $1, %ebp
+; CHECK-NEXT: movzbl %al, %ebx
+; CHECK-NEXT: andl $1, %ebx
; CHECK-NEXT: xorpd %xmm0, %xmm0
+; CHECK-NEXT: xorl %ebp, %ebp
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: xorl %edi, %edi
; CHECK-NEXT: xorpd %xmm1, %xmm1
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_7: # %bb.i28.i
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: cvttsd2si %xmm1, %ecx
-; CHECK-NEXT: cmpl %edx, %ecx
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: addl $2, %edi
+; CHECK-NEXT: cvttsd2si %xmm1, %esi
+; CHECK-NEXT: cmpl %edx, %esi
+; CHECK-NEXT: cmovgel %ebp, %esi
+; CHECK-NEXT: addl $2, %eax
; CHECK-NEXT: xorps %xmm2, %xmm2
-; CHECK-NEXT: cvtsi2sd %ecx, %xmm2
+; CHECK-NEXT: cvtsi2sd %esi, %xmm2
; CHECK-NEXT: xorpd %xmm1, %xmm1
; CHECK-NEXT: subsd %xmm2, %xmm1
; CHECK-NEXT: mulsd %xmm0, %xmm1
-; CHECK-NEXT: addl $-2, %ebp
+; CHECK-NEXT: addl $-2, %ebx
; CHECK-NEXT: jne .LBB0_7
; CHECK-NEXT: # %bb.8: # %mp_unexp_d2mp.exit29.i
; CHECK-NEXT: movl $0, 0
; CHECK-NEXT: je .LBB0_9
; CHECK-NEXT: # %bb.10: # %mp_sqrt_init.exit
-; CHECK-NEXT: movl %esi, (%esp) # 4-byte Spill
+; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: movl %edx, %edi
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; CHECK-NEXT: movl %ebx, %edx
+; CHECK-NEXT: movl %edx, (%esp) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-NEXT: calll mp_mul_csqu at PLT
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: movl $-1, %edx
@@ -67,18 +67,17 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
; CHECK-NEXT: calll rdft at PLT
; CHECK-NEXT: addl $12, %esp
; CHECK-NEXT: xorl %ecx, %ecx
+; CHECK-NEXT: movl (%esp), %edx # 4-byte Reload
; CHECK-NEXT: movl %edi, %esi
-; CHECK-NEXT: movl %edi, %edx
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
; CHECK-NEXT: pushl $0
; CHECK-NEXT: calll mp_mul_d2i at PLT
; CHECK-NEXT: addl $12, %esp
-; CHECK-NEXT: testl %ebp, %ebp
+; CHECK-NEXT: testl %ebx, %ebx
; CHECK-NEXT: je .LBB0_11
; CHECK-NEXT: .LBB0_3: # %cond_true.i
-; CHECK-NEXT: addl $4, %esp
+; CHECK-NEXT: addl $8, %esp
; CHECK-NEXT: .LBB0_4: # %cond_true.i
; CHECK-NEXT: popl %esi
; CHECK-NEXT: popl %edi
@@ -91,39 +90,39 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
; CHECK-NEXT: jmp .LBB0_9
; CHECK-NEXT: .LBB0_11: # %cond_false.i
; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: movl %esi, %edx
+; CHECK-NEXT: movl (%esp), %edx # 4-byte Reload
; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
; CHECK-NEXT: pushl $0
; CHECK-NEXT: calll mp_round at PLT
; CHECK-NEXT: addl $8, %esp
; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl %esi
; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: pushl %ebx
; CHECK-NEXT: calll mp_add at PLT
; CHECK-NEXT: addl $12, %esp
; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: pushl %esi
; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
; CHECK-NEXT: calll mp_sub at PLT
; CHECK-NEXT: addl $12, %esp
; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT: pushl %edi
; CHECK-NEXT: pushl $0
; CHECK-NEXT: calll mp_round at PLT
; CHECK-NEXT: addl $8, %esp
; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT: pushl %esi
; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
; CHECK-NEXT: pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; CHECK-NEXT: calll mp_mul_d2i at PLT
-; CHECK-NEXT: addl $16, %esp
+; CHECK-NEXT: addl $20, %esp
; CHECK-NEXT: jmp .LBB0_4
entry:
br label %bb.i5
diff --git a/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll b/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
index 6f1419c538eb7..45b701307a3fd 100644
--- a/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
@@ -19,9 +19,8 @@ define i16 @SQLDriversW(ptr %henv, i16 zeroext %fDir, ptr %szDrvDesc, i16 signe
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: je LBB0_1
; CHECK-NEXT: ## %bb.3: ## %bb28
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
@@ -36,16 +35,18 @@ define i16 @SQLDriversW(ptr %henv, i16 zeroext %fDir, ptr %szDrvDesc, i16 signe
; CHECK-NEXT: subl $12, %esp
; CHECK-NEXT: movzwl %di, %eax
; CHECK-NEXT: movswl %bp, %ebp
-; CHECK-NEXT: movswl %bx, %edi
+; CHECK-NEXT: movswl %bx, %ebx
; CHECK-NEXT: pushl $87
-; CHECK-NEXT: pushl %edx
+; CHECK-NEXT: movl %esi, %edi
+; CHECK-NEXT: pushl %esi
; CHECK-NEXT: pushl %ebp
; CHECK-NEXT: pushl $0
-; CHECK-NEXT: movl %esi, %ebx
-; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: pushl %edx
+; CHECK-NEXT: pushl %ebx
; CHECK-NEXT: pushl $0
; CHECK-NEXT: pushl %eax
+; CHECK-NEXT: movl %ecx, %ebx
; CHECK-NEXT: pushl %ecx
; CHECK-NEXT: calll _SQLDrivers_Internal
; CHECK-NEXT: addl $48, %esp
@@ -53,38 +54,38 @@ define i16 @SQLDriversW(ptr %henv, i16 zeroext %fDir, ptr %szDrvDesc, i16 signe
; CHECK-NEXT: testb %cl, %cl
; CHECK-NEXT: je LBB0_1
; CHECK-NEXT: ## %bb.5:
-; CHECK-NEXT: movl %ebx, %esi
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movl %ebx, %ecx
+; CHECK-NEXT: movl %edi, %esi
; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
; CHECK-NEXT: LBB0_6: ## %done
-; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: testb %dl, %dl
; CHECK-NEXT: je LBB0_7
; CHECK-NEXT: ## %bb.8: ## %bb167
; CHECK-NEXT: subl $12, %esp
; CHECK-NEXT: movl L_iodbcdm_global_lock$non_lazy_ptr, %ecx
; CHECK-NEXT: pushl %ecx
-; CHECK-NEXT: movl %eax, %esi
+; CHECK-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
; CHECK-NEXT: calll _pthread_mutex_unlock
; CHECK-NEXT: addl $16, %esp
-; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 2-byte Folded Reload
; CHECK-NEXT: jmp LBB0_2
; CHECK-NEXT: LBB0_7: ## %bb150
; CHECK-NEXT: cwtl
; CHECK-NEXT: subl $8, %esp
-; CHECK-NEXT: movswl %bp, %ecx
+; CHECK-NEXT: movswl %bp, %edx
; CHECK-NEXT: movswl %bx, %ebx
; CHECK-NEXT: movzwl %di, %edi
+; CHECK-NEXT: pushl %esi
; CHECK-NEXT: pushl %edx
-; CHECK-NEXT: pushl %ecx
; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
-; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
; CHECK-NEXT: pushl %ebx
; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT: pushl %ecx
; CHECK-NEXT: pushl %eax
; CHECK-NEXT: pushl $1
; CHECK-NEXT: calll _trace_SQLDriversW
diff --git a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
index fca0ca1bbd09f..3ce244cd19d27 100644
--- a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
@@ -17,28 +17,20 @@
define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct.wxString) %agg.result, ptr %this, ptr %format, ptr %tz, i1 %foo) personality ptr @__gxx_personality_v0 {
; CHECK-LABEL: _ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE:
; CHECK: ## %bb.0: ## %entry
-; CHECK-NEXT: pushl %ebp
-; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: pushl %ebx
-; CHECK-NEXT: .cfi_def_cfa_offset 12
-; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: .cfi_def_cfa_offset 20
-; CHECK-NEXT: subl $28, %esp
-; CHECK-NEXT: .cfi_def_cfa_offset 48
-; CHECK-NEXT: .cfi_offset %esi, -20
-; CHECK-NEXT: .cfi_offset %edi, -16
-; CHECK-NEXT: .cfi_offset %ebx, -12
-; CHECK-NEXT: .cfi_offset %ebp, -8
+; CHECK-NEXT: .cfi_def_cfa_offset 12
+; CHECK-NEXT: subl $20, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: .cfi_offset %esi, -12
+; CHECK-NEXT: .cfi_offset %ebx, -8
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
; CHECK-NEXT: testb $1, %bl
; CHECK-NEXT: je LBB0_21
; CHECK-NEXT: ## %bb.1:
-; CHECK-NEXT: movl L_.str89$non_lazy_ptr, %edi
; CHECK-NEXT: movb $1, %bh
-; CHECK-NEXT: movl L_.str$non_lazy_ptr, %ebp
; CHECK-NEXT: jmp LBB0_2
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: LBB0_17: ## %bb7806
@@ -56,7 +48,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct
; CHECK-NEXT: ## %bb.3: ## %bb3306
; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: Ltmp0: ## EH_LABEL
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; CHECK-NEXT: movl L_.str89$non_lazy_ptr, %eax
+; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp)
; CHECK-NEXT: movl $0, (%esp)
; CHECK-NEXT: calll __ZN12wxStringBaseaSEPKw
; CHECK-NEXT: Ltmp1: ## EH_LABEL
@@ -99,7 +92,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct
; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp)
; CHECK-NEXT: movl L__ZZNK10wxDateTime5GetTmERKNS_8TimeZoneEE12__FUNCTION__$non_lazy_ptr, %eax
; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %ebp, (%esp)
+; CHECK-NEXT: movl L_.str$non_lazy_ptr, %eax
+; CHECK-NEXT: movl %eax, (%esp)
; CHECK-NEXT: movl $1717, {{[0-9]+}}(%esp) ## imm = 0x6B5
; CHECK-NEXT: calll __Z10wxOnAssertPKwiPKcS0_S0_
; CHECK-NEXT: Ltmp7: ## EH_LABEL
@@ -145,11 +139,9 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct
; CHECK-NEXT: Ltmp3: ## EH_LABEL
; CHECK-NEXT: LBB0_21: ## %bb115.critedge.i
; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: addl $28, %esp
+; CHECK-NEXT: addl $20, %esp
; CHECK-NEXT: popl %esi
-; CHECK-NEXT: popl %edi
; CHECK-NEXT: popl %ebx
-; CHECK-NEXT: popl %ebp
; CHECK-NEXT: retl $4
; CHECK-NEXT: LBB0_19: ## %lpad.loopexit.split-lp
; CHECK-NEXT: Ltmp15: ## EH_LABEL
diff --git a/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll b/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll
index 283e7b3454f6c..d17824457eb2f 100644
--- a/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll
+++ b/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll
@@ -4,57 +4,52 @@
define fastcc i64 @foo(ptr %p0, ptr %p1, i64 %p2, i64 %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, i64 %p9, i1 %cond, i32 %n) nounwind {
; CHECK-LABEL: foo:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: subq $56, %rsp
; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; CHECK-NEXT: testb $1, %al
-; CHECK-NEXT: jne .LBB0_7
+; CHECK-NEXT: jne .LBB0_1
; CHECK-NEXT: # %bb.2: # %bb1
-; CHECK-NEXT: jne .LBB0_7
+; CHECK-NEXT: jne .LBB0_1
; CHECK-NEXT: # %bb.4: # %bb2
; CHECK-NEXT: movb $1, %r10b
; CHECK-NEXT: testb %r10b, %r10b
-; CHECK-NEXT: jne .LBB0_7
+; CHECK-NEXT: jne .LBB0_1
; CHECK-NEXT: # %bb.5: # %bb2
; CHECK-NEXT: xorl %r10d, %r10d
; CHECK-NEXT: testb %r10b, %r10b
-; CHECK-NEXT: jne .LBB0_9
+; CHECK-NEXT: jne .LBB0_8
; CHECK-NEXT: # %bb.6: # %bb2
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: .LBB0_7: # %bb3
+; CHECK-NEXT: .LBB0_1:
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: .LBB0_8: # %cleanup
; CHECK-NEXT: addq $56, %rsp
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: popq %r14
; CHECK-NEXT: retq
-; CHECK-NEXT: .LBB0_9: # %bb5
+; CHECK-NEXT: .LBB0_8: # %bb5
; CHECK-NEXT: testb $1, %al
-; CHECK-NEXT: jne .LBB0_7
-; CHECK-NEXT: # %bb.10: # %bb7
-; CHECK-NEXT: jne .LBB0_7
-; CHECK-NEXT: # %bb.12: # %bb9
-; CHECK-NEXT: jne .LBB0_7
-; CHECK-NEXT: # %bb.14: # %bb10
-; CHECK-NEXT: jne .LBB0_7
-; CHECK-NEXT: # %bb.16: # %bb11
-; CHECK-NEXT: jne .LBB0_7
-; CHECK-NEXT: # %bb.18: # %bb12
-; CHECK-NEXT: je .LBB0_7
-; CHECK-NEXT: # %bb.19: # %bb13
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: # %bb.9: # %bb7
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: # %bb.11: # %bb9
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: # %bb.13: # %bb10
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: # %bb.15: # %bb11
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: # %bb.17: # %bb12
+; CHECK-NEXT: je .LBB0_1
+; CHECK-NEXT: # %bb.18: # %bb13
; CHECK-NEXT: movb $1, %al
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: je .LBB0_7
-; CHECK-NEXT: # %bb.20: # %bb15
+; CHECK-NEXT: je .LBB0_1
+; CHECK-NEXT: # %bb.19: # %bb15
; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
; CHECK-NEXT: testl %eax, %eax
-; CHECK-NEXT: je .LBB0_7
-; CHECK-NEXT: # %bb.21: # %bb15
+; CHECK-NEXT: je .LBB0_1
+; CHECK-NEXT: # %bb.20: # %bb15
; CHECK-NEXT: cmpl $2, %eax
-; CHECK-NEXT: je .LBB0_7
-; CHECK-NEXT: # %bb.22: # %bb18
+; CHECK-NEXT: je .LBB0_1
+; CHECK-NEXT: # %bb.21: # %bb18
; CHECK-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -65,27 +60,27 @@ define fastcc i64 @foo(ptr %p0, ptr %p1, i64 %p2, i64 %p3, ptr %p4, ptr %p5, ptr
; CHECK-NEXT: movq %rax, (%rsp) # 8-byte Spill
; CHECK-NEXT: movb $1, %al
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_27
-; CHECK-NEXT: # %bb.23:
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; CHECK-NEXT: xorl %ebx, %ebx
-; CHECK-NEXT: .LBB0_24: # %loop
+; CHECK-NEXT: jne .LBB0_25
+; CHECK-NEXT: .LBB0_22: # %loop
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: callq *%rbx
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: callq *%rax
+; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-NEXT: xorl %esi, %esi
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: xorl %r8d, %r8d
-; CHECK-NEXT: movq %r14, %r9
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r9
; CHECK-NEXT: pushq $0
; CHECK-NEXT: pushq $0
; CHECK-NEXT: pushq $0
; CHECK-NEXT: pushq $0
-; CHECK-NEXT: callq *%rbx
+; CHECK-NEXT: callq *%rax
+; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: addq $32, %rsp
; CHECK-NEXT: xorl %edi, %edi
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
@@ -97,22 +92,23 @@ define fastcc i64 @foo(ptr %p0, ptr %p1, i64 %p2, i64 %p3, ptr %p4, ptr %p5, ptr
; CHECK-NEXT: pushq $0
; CHECK-NEXT: pushq $0
; CHECK-NEXT: pushq $0
-; CHECK-NEXT: callq *%rbx
+; CHECK-NEXT: callq *%rax
; CHECK-NEXT: addq $32, %rsp
; CHECK-NEXT: testb $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT: je .LBB0_24
-; CHECK-NEXT: # %bb.25: # %cleanup2
-; CHECK-NEXT: jne .LBB0_27
-; CHECK-NEXT: # %bb.26: # %bb21
+; CHECK-NEXT: je .LBB0_22
+; CHECK-NEXT: # %bb.23: # %cleanup2
+; CHECK-NEXT: jne .LBB0_25
+; CHECK-NEXT: # %bb.24: # %bb21
; CHECK-NEXT: xorl %edi, %edi
; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; CHECK-NEXT: movq %rbx, %rdx
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; CHECK-NEXT: callq memcpy at PLT
-; CHECK-NEXT: movq %rbx, (%rsp) # 8-byte Spill
-; CHECK-NEXT: .LBB0_27: # %exit
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, (%rsp) # 8-byte Spill
+; CHECK-NEXT: .LBB0_25: # %exit
; CHECK-NEXT: movq (%rsp), %rax # 8-byte Reload
-; CHECK-NEXT: jmp .LBB0_8
+; CHECK-NEXT: addq $56, %rsp
+; CHECK-NEXT: retq
entry:
br i1 %cond, label %cleanup, label %bb1
diff --git a/llvm/test/CodeGen/X86/atom-fixup-lea2.ll b/llvm/test/CodeGen/X86/atom-fixup-lea2.ll
index adea9e1386a90..f22d097afc802 100644
--- a/llvm/test/CodeGen/X86/atom-fixup-lea2.ll
+++ b/llvm/test/CodeGen/X86/atom-fixup-lea2.ll
@@ -1,9 +1,9 @@
; RUN: llc < %s -mcpu=atom -mtriple=i686-linux | FileCheck %s
; CHECK:%bb.5
+; CHECK-NEXT:addl
; CHECK-NEXT:leal
-; CHECK-NEXT:leal
-; CHECK-NEXT:leal
+; CHECK-NEXT:addl
; CHECK-NEXT:movl
diff --git a/llvm/test/CodeGen/X86/call-rv-marker.ll b/llvm/test/CodeGen/X86/call-rv-marker.ll
index 8d363bc451c63..2bb3b49c2bd51 100644
--- a/llvm/test/CodeGen/X86/call-rv-marker.ll
+++ b/llvm/test/CodeGen/X86/call-rv-marker.ll
@@ -85,14 +85,11 @@ entry:
define void @rv_marker_3() personality ptr @__gxx_personality_v0 {
; CHECK-LABEL: rv_marker_3
-; CHECK: pushq %r14
+; CHECK: pushq %rbx
; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 24
-; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: subq $16, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: .cfi_offset %rbx, -24
-; CHECK-NEXT: .cfi_offset %r14, -16
+; CHECK-NEXT: .cfi_offset %rbx, -16
; CHECK-NEXT: callq _foo1
; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: callq _objc_retainAutoreleasedReturnValue
diff --git a/llvm/test/CodeGen/X86/csr-split.ll b/llvm/test/CodeGen/X86/csr-split.ll
index 20d577efd2714..03023f3e639e5 100644
--- a/llvm/test/CodeGen/X86/csr-split.ll
+++ b/llvm/test/CodeGen/X86/csr-split.ll
@@ -76,13 +76,12 @@ define dso_local signext i32 @test2(ptr %p1) local_unnamed_addr {
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: retq
; X64-NEXT: .LBB1_3: # %if.then2
-; X64-NEXT: pushq %rbx
+; X64-NEXT: pushq %rax
; X64-NEXT: .cfi_def_cfa_offset 16
-; X64-NEXT: .cfi_offset %rbx, -16
-; X64-NEXT: movq %rdi, %rbx
+; X64-NEXT: movq %rdi, (%rsp) # 8-byte Spill
; X64-NEXT: callq callVoid at PLT
-; X64-NEXT: movq %rbx, %rdi
-; X64-NEXT: popq %rbx
+; X64-NEXT: movq (%rsp), %rdi # 8-byte Reload
+; X64-NEXT: popq %rax
; X64-NEXT: .cfi_def_cfa_offset 8
; X64-NEXT: jmp callNonVoid at PLT # TAILCALL
;
diff --git a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
index 0df3c0b03dc33..ed1cf42af3348 100644
--- a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
+++ b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
@@ -7,26 +7,14 @@
define double @main(i224 %0) #0 {
; CHECK-LABEL: main:
; CHECK: # %bb.0: # %entryitofp-entry
-; CHECK-NEXT: pushq %rbp
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: pushq %r15
-; CHECK-NEXT: .cfi_def_cfa_offset 24
; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: pushq %r13
-; CHECK-NEXT: .cfi_def_cfa_offset 40
-; CHECK-NEXT: pushq %r12
-; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 56
-; CHECK-NEXT: subq $88, %rsp
+; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: subq $120, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 144
-; CHECK-NEXT: .cfi_offset %rbx, -56
-; CHECK-NEXT: .cfi_offset %r12, -48
-; CHECK-NEXT: .cfi_offset %r13, -40
-; CHECK-NEXT: .cfi_offset %r14, -32
-; CHECK-NEXT: .cfi_offset %r15, -24
-; CHECK-NEXT: .cfi_offset %rbp, -16
+; CHECK-NEXT: .cfi_offset %rbx, -24
+; CHECK-NEXT: .cfi_offset %r14, -16
; CHECK-NEXT: movl %ecx, %eax
; CHECK-NEXT: movq %rdi, %r8
; CHECK-NEXT: orq %rdx, %r8
@@ -104,39 +92,45 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: .LBB0_5: # %itofp-sw-default
; CHECK-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rdx, (%rsp)
; CHECK-NEXT: movl %ecx, %r8d
-; CHECK-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %r8, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movb $-87, %r8b
; CHECK-NEXT: subb %al, %r8b
; CHECK-NEXT: movb %r8b, %bl
; CHECK-NEXT: shrb $6, %bl
-; CHECK-NEXT: movzbl %bl, %r12d
+; CHECK-NEXT: movzbl %bl, %r14d
; CHECK-NEXT: movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq $0, (%rsp)
-; CHECK-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq -24(%rsp,%r12,8), %rbx
-; CHECK-NEXT: movq -32(%rsp,%r12,8), %r13
-; CHECK-NEXT: movq %rcx, %rbp
+; CHECK-NEXT: movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq 8(%rsp,%r14,8), %rbx
+; CHECK-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq (%rsp,%r14,8), %rbx
+; CHECK-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movb %r8b, %cl
-; CHECK-NEXT: movq %r13, %r14
-; CHECK-NEXT: shrdq %cl, %rbx, %r14
-; CHECK-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq -48(%rsp,%r12,8), %r15
-; CHECK-NEXT: movq -40(%rsp,%r12,8), %r12
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT: shrdq %cl, %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: movq -16(%rsp,%r14,8), %rcx
+; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq -8(%rsp,%r14,8), %r14
; CHECK-NEXT: movb %r8b, %cl
-; CHECK-NEXT: movq %r12, %r14
-; CHECK-NEXT: shrdq %cl, %r13, %r14
+; CHECK-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT: shrdq %cl, %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
; CHECK-NEXT: movb %r8b, %cl
-; CHECK-NEXT: shrq %cl, %rbx
+; CHECK-NEXT: shrq %cl, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
; CHECK-NEXT: movb %r8b, %cl
-; CHECK-NEXT: shrdq %cl, %r12, %r15
+; CHECK-NEXT: shrdq %cl, %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
; CHECK-NEXT: addb $55, %al
; CHECK-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rbp, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movq $0, -{{[0-9]+}}(%rsp)
@@ -146,17 +140,17 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: andb $24, %cl
; CHECK-NEXT: negb %cl
; CHECK-NEXT: movsbq %cl, %rdx
-; CHECK-NEXT: movq -80(%rsp,%rdx), %rsi
-; CHECK-NEXT: movq -72(%rsp,%rdx), %rdi
-; CHECK-NEXT: movq -64(%rsp,%rdx), %r8
+; CHECK-NEXT: movq -48(%rsp,%rdx), %rsi
+; CHECK-NEXT: movq -40(%rsp,%rdx), %rdi
+; CHECK-NEXT: movq -32(%rsp,%rdx), %r8
; CHECK-NEXT: movb %al, %cl
-; CHECK-NEXT: movq %r8, %r12
-; CHECK-NEXT: shldq %cl, %rdi, %r12
+; CHECK-NEXT: movq %r8, %rbx
+; CHECK-NEXT: shldq %cl, %rdi, %rbx
; CHECK-NEXT: movb %al, %cl
-; CHECK-NEXT: movq %rsi, %r13
-; CHECK-NEXT: shlq %cl, %r13
-; CHECK-NEXT: orq %r12, %r13
-; CHECK-NEXT: movq -56(%rsp,%rdx), %rdx
+; CHECK-NEXT: movq %rsi, %r14
+; CHECK-NEXT: shlq %cl, %r14
+; CHECK-NEXT: orq %rbx, %r14
+; CHECK-NEXT: movq -24(%rsp,%rdx), %rdx
; CHECK-NEXT: movb %al, %cl
; CHECK-NEXT: shldq %cl, %r8, %rdx
; CHECK-NEXT: movl %edx, %edx
@@ -164,13 +158,13 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: shldq %cl, %rsi, %rdi
; CHECK-NEXT: orq %rdx, %rdi
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: orq %rdi, %r13
+; CHECK-NEXT: orq %rdi, %r14
; CHECK-NEXT: setne %al
-; CHECK-NEXT: orq %rax, %r15
-; CHECK-NEXT: movq %r15, %rdi
-; CHECK-NEXT: movq %r14, %rsi
+; CHECK-NEXT: orq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; CHECK-NEXT: movq %rbx, %rcx
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; CHECK-NEXT: jmp .LBB0_6
; CHECK-NEXT: .LBB0_6: # %itofp-sw-epilog
; CHECK-NEXT: movl %edi, %eax
@@ -210,7 +204,7 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: andb $24, %cl
; CHECK-NEXT: negb %cl
; CHECK-NEXT: movsbq %cl, %rcx
-; CHECK-NEXT: movq 48(%rsp,%rcx), %rdx
+; CHECK-NEXT: movq 80(%rsp,%rcx), %rdx
; CHECK-NEXT: movb %al, %cl
; CHECK-NEXT: shlq %cl, %rdx
; CHECK-NEXT: movq %rdx, %rax
@@ -229,19 +223,11 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: orq %rdx, %rax
; CHECK-NEXT: movq %rax, %xmm0
; CHECK-NEXT: .LBB0_10: # %itofp-return
-; CHECK-NEXT: addq $88, %rsp
-; CHECK-NEXT: .cfi_def_cfa_offset 56
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 48
-; CHECK-NEXT: popq %r12
-; CHECK-NEXT: .cfi_def_cfa_offset 40
-; CHECK-NEXT: popq %r13
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: popq %r14
+; CHECK-NEXT: addq $120, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 24
-; CHECK-NEXT: popq %r15
+; CHECK-NEXT: popq %rbx
; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: popq %r14
; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
index 4347d626e5838..f24e0f8283bc9 100644
--- a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
@@ -43,8 +43,8 @@ define i32 @eflagsLiveInPrologue() #0 {
; ENABLE-NEXT: cmovnel %edx, %ecx
; ENABLE-NEXT: movl L_e$non_lazy_ptr, %edx
; ENABLE-NEXT: movb %cl, (%edx)
-; ENABLE-NEXT: leal 1(%ecx), %esi
; ENABLE-NEXT: cltd
+; ENABLE-NEXT: leal 1(%ecx), %esi
; ENABLE-NEXT: idivl %esi
; ENABLE-NEXT: movl L_c$non_lazy_ptr, %eax
; ENABLE-NEXT: movl %edx, (%eax)
@@ -81,8 +81,8 @@ define i32 @eflagsLiveInPrologue() #0 {
; DISABLE-NEXT: cmovnel %edx, %ecx
; DISABLE-NEXT: movl L_e$non_lazy_ptr, %edx
; DISABLE-NEXT: movb %cl, (%edx)
-; DISABLE-NEXT: leal 1(%ecx), %esi
; DISABLE-NEXT: cltd
+; DISABLE-NEXT: leal 1(%ecx), %esi
; DISABLE-NEXT: idivl %esi
; DISABLE-NEXT: movl L_c$non_lazy_ptr, %eax
; DISABLE-NEXT: movl %edx, (%eax)
diff --git a/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll b/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll
index b9afb99697354..92ab8af129d40 100755
--- a/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll
+++ b/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll
@@ -26,8 +26,8 @@ define void @test() personality ptr @__gxx_personality_v0 {
; CHECK-NEXT: .cfi_def_cfa_offset 48
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: .cfi_def_cfa_offset 56
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NEXT: subq $24, %rsp
+; CHECK-NEXT: .cfi_def_cfa_offset 80
; CHECK-NEXT: .cfi_offset %rbx, -56
; CHECK-NEXT: .cfi_offset %r12, -48
; CHECK-NEXT: .cfi_offset %r13, -40
@@ -43,12 +43,12 @@ define void @test() personality ptr @__gxx_personality_v0 {
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movq externalC at GOTPCREL(%rip), %rax
; CHECK-NEXT: movl (%rax), %eax
-; CHECK-NEXT: movl %eax, (%rsp) # 4-byte Spill
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: #APP
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: .Ltmp2:
+; CHECK-NEXT: .Ltmp2: # EH_LABEL
; CHECK-NEXT: callq maythrow at PLT
-; CHECK-NEXT: .Ltmp3:
+; CHECK-NEXT: .Ltmp3: # EH_LABEL
; CHECK-NEXT: jmp .LBB0_4
; CHECK-NEXT: .LBB0_3: # %branchB
; CHECK-NEXT: movq externalB at GOTPCREL(%rip), %rax
@@ -56,14 +56,14 @@ define void @test() personality ptr @__gxx_personality_v0 {
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movq externalD at GOTPCREL(%rip), %rax
; CHECK-NEXT: movl (%rax), %eax
-; CHECK-NEXT: movl %eax, (%rsp) # 4-byte Spill
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: #APP
; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT: .Ltmp0: # EH_LABEL
; CHECK-NEXT: callq maythrow at PLT
-; CHECK-NEXT: .Ltmp1:
+; CHECK-NEXT: .Ltmp1: # EH_LABEL
; CHECK-NEXT: .LBB0_4: # %end
-; CHECK-NEXT: addq $8, %rsp
+; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 56
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: .cfi_def_cfa_offset 48
@@ -79,13 +79,13 @@ define void @test() personality ptr @__gxx_personality_v0 {
; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
; CHECK-NEXT: .LBB0_2: # %lpad
-; CHECK-NEXT: .cfi_def_cfa_offset 64
-; CHECK-NEXT: .Ltmp4:
-; CHECK-NEXT: movq %rax, %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 80
+; CHECK-NEXT: .Ltmp4: # EH_LABEL
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT: addl (%rsp), %edi # 4-byte Folded Reload
+; CHECK-NEXT: addl {{[-0-9]+}}(%r{{[sb]}}p), %edi # 4-byte Folded Reload
; CHECK-NEXT: callq cleanup at PLT
-; CHECK-NEXT: movq %rbx, %rdi
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-NEXT: callq _Unwind_Resume at PLT
%1 = load i1, ptr @external_bool
br i1 %1, label %branchA, label %branchB
diff --git a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
index 0d8dc274da1f2..5f1b34897a3ef 100644
--- a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -94,23 +94,23 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jne LBB0_11
; CHECK-NEXT: ## %bb.12: ## %while.body200.preheader
-; CHECK-NEXT: xorl %r13d, %r13d
+; CHECK-NEXT: xorl %r15d, %r15d
; CHECK-NEXT: leaq LJTI0_0(%rip), %rdx
-; CHECK-NEXT: leaq LJTI0_1(%rip), %r14
+; CHECK-NEXT: leaq LJTI0_1(%rip), %r13
; CHECK-NEXT: movb $1, %sil
; CHECK-NEXT: movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
-; CHECK-NEXT: xorl %r12d, %r12d
+; CHECK-NEXT: xorl %r14d, %r14d
; CHECK-NEXT: jmp LBB0_13
; CHECK-NEXT: LBB0_43: ## %while.cond1037.preheader
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: testb %r13b, %r13b
+; CHECK-NEXT: testb %r15b, %r15b
; CHECK-NEXT: je LBB0_54
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: LBB0_20: ## %while.cond197.backedge
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
; CHECK-NEXT: decl %ebp
; CHECK-NEXT: testl %ebp, %ebp
-; CHECK-NEXT: movl %ebx, %r12d
+; CHECK-NEXT: movl %ebx, %r14d
; CHECK-NEXT: jle LBB0_21
; CHECK-NEXT: LBB0_13: ## %while.body200
; CHECK-NEXT: ## =>This Loop Header: Depth=1
@@ -121,18 +121,18 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: ja LBB0_14
; CHECK-NEXT: ## %bb.55: ## %while.body200
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: movslq (%r14,%rax,4), %rax
-; CHECK-NEXT: addq %r14, %rax
+; CHECK-NEXT: movslq (%r13,%rax,4), %rax
+; CHECK-NEXT: addq %r13, %rax
; CHECK-NEXT: jmpq *%rax
; CHECK-NEXT: LBB0_25: ## %sw.bb474
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: testb %r13b, %r13b
-; CHECK-NEXT: ## implicit-def: $r15
+; CHECK-NEXT: testb %r15b, %r15b
+; CHECK-NEXT: ## implicit-def: $r12
; CHECK-NEXT: jne LBB0_33
; CHECK-NEXT: ## %bb.26: ## %do.body479.preheader
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: testb %r13b, %r13b
-; CHECK-NEXT: ## implicit-def: $r15
+; CHECK-NEXT: testb %r15b, %r15b
+; CHECK-NEXT: ## implicit-def: $r12
; CHECK-NEXT: jne LBB0_33
; CHECK-NEXT: ## %bb.27: ## %land.rhs485.preheader
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
@@ -141,8 +141,8 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: LBB0_31: ## %do.body479.backedge
; CHECK-NEXT: ## in Loop: Header=BB0_28 Depth=2
-; CHECK-NEXT: leaq 1(%r15), %rax
-; CHECK-NEXT: testb %r13b, %r13b
+; CHECK-NEXT: leaq 1(%r12), %rax
+; CHECK-NEXT: testb %r15b, %r15b
; CHECK-NEXT: je LBB0_32
; CHECK-NEXT: LBB0_28: ## %land.rhs485
; CHECK-NEXT: ## Parent Loop BB0_13 Depth=1
@@ -151,7 +151,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: jne LBB0_54
; CHECK-NEXT: ## %bb.29: ## %cond.true.i.i2780
; CHECK-NEXT: ## in Loop: Header=BB0_28 Depth=2
-; CHECK-NEXT: movq %rax, %r15
+; CHECK-NEXT: movq %rax, %r12
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jne LBB0_31
; CHECK-NEXT: ## %bb.30: ## %lor.rhs500
@@ -197,10 +197,10 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: jmp LBB0_20
; CHECK-NEXT: LBB0_32: ## %if.end517.loopexitsplit
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: incq %r15
+; CHECK-NEXT: incq %r12
; CHECK-NEXT: LBB0_33: ## %if.end517
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: leal -324(%r12), %eax
+; CHECK-NEXT: leal -324(%r14), %eax
; CHECK-NEXT: cmpl $59, %eax
; CHECK-NEXT: ja LBB0_34
; CHECK-NEXT: ## %bb.56: ## %if.end517
@@ -210,11 +210,11 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: jb LBB0_37
; CHECK-NEXT: LBB0_34: ## %if.end517
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: cmpl $11, %r12d
+; CHECK-NEXT: cmpl $11, %r14d
; CHECK-NEXT: je LBB0_37
; CHECK-NEXT: ## %bb.35: ## %if.end517
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT: cmpl $24, %r12d
+; CHECK-NEXT: cmpl $24, %r14d
; CHECK-NEXT: je LBB0_37
; CHECK-NEXT: ## %bb.36: ## %if.then532
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
@@ -224,12 +224,12 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
; CHECK-NEXT: LBB0_37: ## %for.cond534
; CHECK-NEXT: ## Parent Loop BB0_13 Depth=1
; CHECK-NEXT: ## => This Inner Loop Header: Depth=2
-; CHECK-NEXT: testb %r13b, %r13b
+; CHECK-NEXT: testb %r15b, %r15b
; CHECK-NEXT: jne LBB0_37
; CHECK-NEXT: ## %bb.38: ## %for.cond542.preheader
; CHECK-NEXT: ## in Loop: Header=BB0_13 Depth=1
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: movb $0, (%r15)
+; CHECK-NEXT: movb $0, (%r12)
; CHECK-NEXT: leaq LJTI0_0(%rip), %rdx
; CHECK-NEXT: jmp LBB0_20
; CHECK-NEXT: .p2align 4
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening.ll b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
index 0fd81cdad4d2b..2773221921944 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
@@ -32,11 +32,8 @@ entry:
define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2, ptr %ptr3) speculative_load_hardening {
; X64-LABEL: test_basic_conditions:
; X64: # %bb.0: # %entry
-; X64-NEXT: pushq %rbx
-; X64-NEXT: .cfi_def_cfa_offset 16
-; X64-NEXT: subq $16, %rsp
+; X64-NEXT: subq $24, %rsp
; X64-NEXT: .cfi_def_cfa_offset 32
-; X64-NEXT: .cfi_offset %rbx, -16
; X64-NEXT: movq %rsp, %rax
; X64-NEXT: movq $-1, %r10
; X64-NEXT: sarq $63, %rax
@@ -51,9 +48,7 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
; X64-NEXT: .LBB1_8: # %exit
; X64-NEXT: shlq $47, %rax
; X64-NEXT: orq %rax, %rsp
-; X64-NEXT: addq $16, %rsp
-; X64-NEXT: .cfi_def_cfa_offset 16
-; X64-NEXT: popq %rbx
+; X64-NEXT: addq $24, %rsp
; X64-NEXT: .cfi_def_cfa_offset 8
; X64-NEXT: retq
; X64-NEXT: .LBB1_4: # %then2
@@ -76,16 +71,18 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
; X64-NEXT: orq %rax, %rdi
; X64-NEXT: movl (%r8,%rdi,4), %esi
; X64-NEXT: orl %eax, %esi
-; X64-NEXT: movq (%r9), %rbx
-; X64-NEXT: orq %rax, %rbx
-; X64-NEXT: addl (%rbx), %esi
+; X64-NEXT: movq (%r9), %rcx
+; X64-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT: orq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; X64-NEXT: addl (%rcx), %esi
; X64-NEXT: shlq $47, %rax
; X64-NEXT: # kill: def $edi killed $edi killed $rdi
; X64-NEXT: orq %rax, %rsp
; X64-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-NEXT: callq leak at PLT
; X64-NEXT: .Lslh_ret_addr0:
-; X64-NEXT: movq %rbx, %rcx
+; X64-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; X64-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
; X64-NEXT: movq %rsp, %rax
; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rdx
@@ -101,11 +98,8 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
;
; X64-LFENCE-LABEL: test_basic_conditions:
; X64-LFENCE: # %bb.0: # %entry
-; X64-LFENCE-NEXT: pushq %rbx
-; X64-LFENCE-NEXT: .cfi_def_cfa_offset 16
-; X64-LFENCE-NEXT: subq $16, %rsp
+; X64-LFENCE-NEXT: subq $24, %rsp
; X64-LFENCE-NEXT: .cfi_def_cfa_offset 32
-; X64-LFENCE-NEXT: .cfi_offset %rbx, -16
; X64-LFENCE-NEXT: testl %edi, %edi
; X64-LFENCE-NEXT: jne .LBB1_6
; X64-LFENCE-NEXT: # %bb.1: # %then1
@@ -128,21 +122,21 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
; X64-LFENCE-NEXT: addl (%r8), %eax
; X64-LFENCE-NEXT: movslq %eax, %rdi
; X64-LFENCE-NEXT: movl (%r8,%rdi,4), %esi
-; X64-LFENCE-NEXT: movq (%r9), %rbx
-; X64-LFENCE-NEXT: addl (%rbx), %esi
+; X64-LFENCE-NEXT: movq (%r9), %rax
+; X64-LFENCE-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-LFENCE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-LFENCE-NEXT: addl (%rax), %esi
; X64-LFENCE-NEXT: # kill: def $edi killed $edi killed $rdi
; X64-LFENCE-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-LFENCE-NEXT: callq leak at PLT
-; X64-LFENCE-NEXT: movq %rbx, %rax
+; X64-LFENCE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; X64-LFENCE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
; X64-LFENCE-NEXT: .LBB1_5: # %merge
; X64-LFENCE-NEXT: movslq (%rax), %rax
; X64-LFENCE-NEXT: movl $0, (%r8,%rax,4)
; X64-LFENCE-NEXT: .LBB1_6: # %exit
; X64-LFENCE-NEXT: lfence
-; X64-LFENCE-NEXT: addq $16, %rsp
-; X64-LFENCE-NEXT: .cfi_def_cfa_offset 16
-; X64-LFENCE-NEXT: popq %rbx
+; X64-LFENCE-NEXT: addq $24, %rsp
; X64-LFENCE-NEXT: .cfi_def_cfa_offset 8
; X64-LFENCE-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/throws-cfi-fp.ll b/llvm/test/CodeGen/X86/throws-cfi-fp.ll
index c6fc8bda52010..cf5b9e0daad6f 100644
--- a/llvm/test/CodeGen/X86/throws-cfi-fp.ll
+++ b/llvm/test/CodeGen/X86/throws-cfi-fp.ll
@@ -26,12 +26,9 @@ define void @_Z6throwsv() #0 personality ptr @__gxx_personality_v0 {
; PEI-NEXT: frame-setup CFI_INSTRUCTION offset $rbp, -16
; PEI-NEXT: $rbp = frame-setup MOV64rr $rsp
; PEI-NEXT: frame-setup CFI_INSTRUCTION def_cfa_register $rbp
-; PEI-NEXT: frame-setup PUSH64r killed $rbx, implicit-def $rsp, implicit $rsp
-; PEI-NEXT: frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp
-; PEI-NEXT: {{^ +}}CFI_INSTRUCTION offset $rbx, -24
+; PEI-NEXT: $rsp = frame-setup SUB64ri32 $rsp, 16, implicit-def dead $eflags
; PEI: bb.4.try.cont:
-; PEI-NEXT: $rsp = frame-destroy ADD64ri32 $rsp, 8, implicit-def dead $eflags
-; PEI-NEXT: $rbx = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+; PEI-NEXT: $rsp = frame-destroy ADD64ri32 $rsp, 16, implicit-def dead $eflags
; PEI-NEXT: $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp
; PEI-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $rsp, 8
; PEI-NEXT: RET 0
diff --git a/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll b/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll
index 6663d91b75355..795c58f5d7d40 100644
--- a/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll
+++ b/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll
@@ -13,7 +13,7 @@ $__clang_call_terminate = comdat any
define void @_Z6throwsv() personality ptr @__gxx_personality_v0 {
; CHECK-LABEL: _Z6throwsv:
-; CHECK: popq %rbx
+; CHECK: popq %rax
; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
; CHECK-NEXT: .LBB0_1:
More information about the llvm-branch-commits
mailing list