[llvm-branch-commits] [llvm] [RegAlloc] Add enableCSRFirstTimeMinCost to let RegAlloc has more choice (PR #239530)

Piyou Chen via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Thu Nov 12 22:32:40 PST 2026


https://github.com/BeMg updated https://github.com/llvm/llvm-project/pull/239530

>From 06b865788bc7558e44ccffd71f66d4b2174b03fa Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Thu, 5 Nov 2026 23:18:29 -0800
Subject: [PATCH 1/3] [RegAlloc] Make tryAssignCSRFirstTime consider
 Remat/Spill

---
 .../include/llvm/CodeGen/TargetRegisterInfo.h |   8 +
 llvm/lib/CodeGen/RegAllocGreedy.cpp           | 180 +++++++++++++++++-
 llvm/lib/CodeGen/RegAllocGreedy.h             |  15 ++
 3 files changed, 201 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index fc3b4d6dda844..3888f141b5370 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -899,6 +899,14 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
     return 30;
   }
 
+  /// Return true if the greedy register allocator should take
+  /// the lowest-cost choice before using an unused CSR.
+  /// This uses tryAssignCSRFirstTimeMinCost instead of
+  /// tryAssignCSRFirstTime.
+  virtual bool enableCSRFirstTimeMinCost(const MachineFunction &MF) const {
+    return false;
+  }
+
   /// Returns true if the target requires (and can make use of) the register
   /// scavenger.
   virtual bool requiresRegisterScavenging(const MachineFunction &MF) const {
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index f7655e97b9942..acce796d5ff65 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2471,6 +2471,49 @@ BlockFrequency RAGreedy::calcRematCost(const LiveInterval &VirtReg,
   return HasUse ? RematCost : BlockFrequency::max();
 }
 
+/// Return true if the spiller can rematerialize each use of \p LI.
+/// And it can rematerialize for only one def.
+static bool isRematerializableFromOriginal(const LiveInterval &LI,
+                                           const LiveIntervals &LIS,
+                                           const VirtRegMap &VRM,
+                                           const MachineRegisterInfo &MRI,
+                                           const TargetInstrInfo &TII) {
+  const LiveInterval &OrigLI = LIS.getInterval(VRM.getOriginal(LI.reg()));
+  const MachineInstr *Def = nullptr;
+  for (const VNInfo *VNI : OrigLI.valnos) {
+    if (VNI->isUnused())
+      continue;
+    // Maybe more than one value def for VirtReg
+    if (VNI->isPHIDef() || Def)
+      return false;
+    Def = LIS.getInstructionFromIndex(VNI->def);
+    if (!Def || !TII.isReMaterializable(*Def))
+      return false;
+  }
+  if (!Def)
+    return false;
+  for (const MachineInstr &MI : MRI.use_nodbg_instructions(LI.reg()))
+    if (!VirtRegAuxInfo::allUsesAvailableAt(Def, LIS.getInstructionIndex(MI),
+                                            LIS, MRI, TII))
+      return false;
+  return true;
+}
+
+/// The cost of rematerializing \p LI before each of its uses.
+static BlockFrequency calcRematCostAtUses(const LiveInterval &LI,
+                                          const MachineRegisterInfo &MRI,
+                                          const SpillPlacement &SpillPlacer) {
+  uint64_t RematCost = 0;
+  SmallPtrSet<const MachineInstr *, 8> Seen;
+  for (const MachineInstr &MI : MRI.use_nodbg_instructions(LI.reg())) {
+    if (MI.isMetaInstruction() || !Seen.insert(&MI).second)
+      continue;
+    RematCost += SpillPlacer.getBlockFrequency(MI.getParent()->getNumber())
+                     .getFrequency();
+  }
+  return BlockFrequency(RematCost);
+}
+
 /// Using a CSR for the first time has a cost because it causes push|pop
 /// to be added to prologue|epilogue. Splitting a cold section of the live
 /// range can have lower cost than using the CSR for the first time;
@@ -2524,6 +2567,135 @@ MCRegister RAGreedy::tryAssignCSRFirstTime(
   return PhysReg;
 }
 
+// Like tryAssignCSRFirstTime before assign the unused CSR to VirtReg, try
+// other choices. There are four results:
+//   1. Pre-split VirtReg
+//   2. Remat/Spill VirtReg
+//   3. Evict rematerializable live ranges if that reg is not unused CSR.
+//   4. Use the CSR for this VirtReg
+// The goal is using the less cost register.
+MCRegister RAGreedy::tryAssignCSRFirstTimeMinCost(
+    const LiveInterval &VirtReg, AllocationOrder &Order, MCRegister PhysReg,
+    uint8_t &CostPerUseLimit, SmallVectorImpl<Register> &NewVRegs) {
+  // Compute Remat cost.
+  std::optional<BlockFrequency> RematCost;
+  if (VirtReg.isSpillable() &&
+      isRematerializableFromOriginal(VirtReg, *LIS, *VRM, *MRI, *TII))
+    RematCost = calcRematCostAtUses(VirtReg, *MRI, *SpillPlacer);
+
+  std::optional<BlockFrequency> SpillRematCost;
+  if (VirtReg.isSpillable()) {
+    SpillRematCost = calcSpillCost(VirtReg);
+    if (RematCost)
+      SpillRematCost = std::min(*SpillRematCost, *RematCost);
+  }
+
+  LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
+
+  if (Stage == RS_Spill && VirtReg.isSpillable()) {
+    // We choose spill or rematerialize over using the CSR for the first time if
+    // the spill cost is lower than CSRCost.
+    if (*SpillRematCost >= CSRCost)
+      return PhysReg;
+
+    // We are going to spill or rematerialize, set CostPerUseLimit to 1 to make
+    // sure that we will not use a callee-saved register in tryEvict.
+    CostPerUseLimit = 1;
+    return MCRegister();
+  }
+
+  if (Stage < RS_Split) {
+    SA->analyze(&VirtReg);
+    // A pre-split must cost less than CSRCost.
+    // CSRCost is the cost for place the CSR save/restore. And it be set by
+    // target.
+    BlockFrequency SplitCost = CSRCost;
+    unsigned NumCands = 0;
+    unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, SplitCost,
+                                                 NumCands, true /*IgnoreCSR*/);
+
+    bool SpillRematCostsLess = SpillRematCost && *SpillRematCost < SplitCost;
+    if (BestCand != NoCand && !SpillRematCostsLess) {
+      doRegionSplit(VirtReg, BestCand, false /*HasCompact*/, NewVRegs);
+      return MCRegister();
+    }
+  }
+
+  bool SpillCostsLess = SpillRematCost && *SpillRematCost < CSRCost;
+  BlockFrequency EvictLimit = SpillCostsLess ? *SpillRematCost : CSRCost;
+  // Use Evict Reg
+  if (MCRegister Reg =
+          tryEvictRematForCSR(VirtReg, Order, NewVRegs, EvictLimit))
+    return Reg;
+
+  // Use CSR
+  if (!SpillCostsLess)
+    return PhysReg;
+
+  // Use Spill/Remat
+  // We are going to spill, set CostPerUseLimit to 1 to make sure that
+  // we will not use a callee-saved register in tryEvict.
+  ExtraInfo->setStage(VirtReg, RS_Spill);
+  CostPerUseLimit = 1;
+  return MCRegister();
+}
+
+/// Return the cost of evicting and rematerializing all the live ranges that
+/// interfere with \p VirtReg in \p PhysReg.
+/// Return std::nullopt when
+///  1. has lower Cascade than \p Cascade.
+///  2. can't be spilled.
+///  3. can't be rematerialized.
+std::optional<BlockFrequency>
+RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
+                             unsigned Cascade) const {
+  BlockFrequency Cost(0);
+  SmallPtrSet<const LiveInterval *, 4> Seen;
+  for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
+    for (const LiveInterval *Intf :
+         Matrix->query(VirtReg, Unit).interferingVRegs()) {
+      if (!Seen.insert(Intf).second)
+        continue;
+      if (ExtraInfo->getCascade(Intf->reg()) >= Cascade ||
+          !Intf->isSpillable() ||
+          !isRematerializableFromOriginal(*Intf, *LIS, *VRM, *MRI, *TII))
+        return std::nullopt;
+      Cost += calcRematCostAtUses(*Intf, *MRI, *SpillPlacer);
+    }
+  }
+  return Cost;
+}
+
+/// Before \p VirtReg uses a CSR for the first time, look for a register that
+/// is not an unused CSR and that only rematerializable live ranges occupy.
+/// If rematerializing them costs less than \p MaxCost, evict them and
+/// return the register.
+MCRegister RAGreedy::tryEvictRematForCSR(const LiveInterval &VirtReg,
+                                         AllocationOrder &Order,
+                                         SmallVectorImpl<Register> &NewVRegs,
+                                         BlockFrequency MaxCost) {
+  unsigned Cascade = ExtraInfo->getCascadeOrCurrentNext(VirtReg.reg());
+  MCRegister BestReg;
+  BlockFrequency BestCost = MaxCost;
+  for (MCRegister PhysReg : Order.getOrder()) {
+    if (EvictAdvisor->getUnusedCalleeSavedReg(VirtReg, PhysReg) ||
+        Matrix->checkInterference(VirtReg, PhysReg) !=
+            LiveRegMatrix::IK_VirtReg)
+      continue;
+    std::optional<BlockFrequency> Cost =
+        calcEvictRematCost(VirtReg, PhysReg, Cascade);
+    if (Cost && *Cost < BestCost) {
+      BestCost = *Cost;
+      BestReg = PhysReg;
+    }
+  }
+  if (!BestReg)
+    return MCRegister();
+
+  evictInterference(VirtReg, BestReg, NewVRegs);
+  return BestReg;
+}
+
 void RAGreedy::aboutToRemoveInterval(const LiveInterval &LI) {
   // Do not keep invalid information around.
   SetOfBrokenHints.remove(&LI);
@@ -2791,8 +2963,12 @@ MCRegister RAGreedy::selectOrSplitImpl(const LiveInterval &VirtReg,
     if (CSRCost.getFrequency() &&
         EvictAdvisor->getUnusedCalleeSavedReg(VirtReg, PhysReg) &&
         NewVRegs.empty()) {
-      MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
-                                                CostPerUseLimit, NewVRegs);
+      MCRegister CSRReg =
+          TRI->enableCSRFirstTimeMinCost(*MF)
+              ? tryAssignCSRFirstTimeMinCost(VirtReg, Order, PhysReg,
+                                             CostPerUseLimit, NewVRegs)
+              : tryAssignCSRFirstTime(VirtReg, Order, PhysReg, CostPerUseLimit,
+                                      NewVRegs);
       if (CSRReg || !NewVRegs.empty())
         // Return now if we decide to use a CSR or create new vregs due to
         // pre-splitting.
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 138da362a1e76..c9c1d68f623c0 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -365,6 +365,21 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
   /// rematerialization is not possible.
   BlockFrequency calcRematCost(const LiveInterval &VirtReg,
                                AllocationOrder &Order) const;
+  /// Like tryAssignCSRFirstTime, but take the choice with the lowest cost of:
+  /// a pre-split, spilling or rematerializing VirtReg, evicting
+  /// rematerializable live ranges, and the first use of the CSR.
+  MCRegister tryAssignCSRFirstTimeMinCost(const LiveInterval &VirtReg,
+                                          AllocationOrder &Order,
+                                          MCRegister PhysReg,
+                                          uint8_t &CostPerUseLimit,
+                                          SmallVectorImpl<Register> &NewVRegs);
+  std::optional<BlockFrequency>
+  calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
+                     unsigned Cascade) const;
+  MCRegister tryEvictRematForCSR(const LiveInterval &VirtReg,
+                                 AllocationOrder &Order,
+                                 SmallVectorImpl<Register> &NewVRegs,
+                                 BlockFrequency MaxCost);
   BlockFrequency calcSpillCost(const LiveInterval &LI);
   void initializeCSRCost();
   MCRegister tryBlockSplit(const LiveInterval &, AllocationOrder &,

>From 969a76f208926513080756d16ef646d7cc405537 Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Sun, 8 Nov 2026 18:07:25 -0800
Subject: [PATCH 2/3] !fixup fix format

---
 llvm/lib/CodeGen/RegAllocGreedy.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index c9c1d68f623c0..4335343942320 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -373,9 +373,9 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
                                           MCRegister PhysReg,
                                           uint8_t &CostPerUseLimit,
                                           SmallVectorImpl<Register> &NewVRegs);
-  std::optional<BlockFrequency>
-  calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
-                     unsigned Cascade) const;
+  std::optional<BlockFrequency> calcEvictRematCost(const LiveInterval &VirtReg,
+                                                   MCRegister PhysReg,
+                                                   unsigned Cascade) const;
   MCRegister tryEvictRematForCSR(const LiveInterval &VirtReg,
                                  AllocationOrder &Order,
                                  SmallVectorImpl<Register> &NewVRegs,

>From 9babcf5843331848466a488f0840b8778dd7ce00 Mon Sep 17 00:00:00 2001
From: BeMg <gccbg04538 at gmail.com>
Date: Fri, 13 Nov 2026 14:07:07 +0800
Subject: [PATCH 3/3] !fixup Merge the tryAssignCSRFirstTimeMinCost into
 tryAssignCSRFirstTime and make it enable by default
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Note: I still added a hook, shouldSpillInsteadOfFirstCSRUse, to avoid a test case crash. The crash happens because AMDGPU has a register class that isn’t suitable for this kind of spill.

I keep it enable by default but disable for AMDGPU.
---
 .../include/llvm/CodeGen/TargetRegisterInfo.h |  14 +-
 llvm/lib/CodeGen/RegAllocGreedy.cpp           | 161 +++----
 llvm/lib/CodeGen/RegAllocGreedy.h             |  13 +-
 llvm/lib/Target/AMDGPU/SIRegisterInfo.h       |   9 +
 llvm/test/CodeGen/AArch64/call-rv-marker.ll   |  51 +-
 .../test/CodeGen/AArch64/sme-za-exceptions.ll |  47 +-
 llvm/test/CodeGen/AMDGPU/bf16.ll              | 306 ++++++------
 llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll    | 396 +++++++---------
 llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll     | 199 ++++----
 ...nfloop-subrange-spill-inspect-subrange.mir |  40 +-
 .../CodeGen/AMDGPU/infloop-subrange-spill.mir |  37 +-
 llvm/test/CodeGen/AMDGPU/issue48473.mir       |   2 +-
 .../AMDGPU/large-avgpr-assign-last.mir        |  10 +-
 llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll   | 404 ++++++++--------
 llvm/test/CodeGen/AMDGPU/maximumnum.ll        | 447 +++++++++---------
 llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll   | 404 ++++++++--------
 llvm/test/CodeGen/AMDGPU/minimumnum.ll        | 447 +++++++++---------
 llvm/test/CodeGen/AMDGPU/select.f16.ll        | 168 +++----
 llvm/test/CodeGen/RISCV/csr-first-use-cost.ll |  18 +-
 .../RISCV/exception-pointer-register.ll       |  36 +-
 .../X86/2007-08-09-IllegalX86-64Asm.ll        |  24 +-
 .../CodeGen/X86/2007-11-30-LoadFolding-Bug.ll |  73 ++-
 llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll  |  35 +-
 .../CodeGen/X86/2008-04-17-CoalescerBug.ll    |  30 +-
 .../CodeGen/X86/apx/memfold-origVNI-crash.ll  |  90 ++--
 llvm/test/CodeGen/X86/atom-fixup-lea2.ll      |   4 +-
 llvm/test/CodeGen/X86/call-rv-marker.ll       |   9 +-
 llvm/test/CodeGen/X86/csr-split.ll            |   9 +-
 .../CodeGen/X86/expand-large-fp-optnone.ll    | 110 ++---
 llvm/test/CodeGen/X86/i386-shrink-wrapping.ll |   4 +-
 llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll  |  28 +-
 llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll |  40 +-
 .../CodeGen/X86/speculative-load-hardening.ll |  36 +-
 llvm/test/CodeGen/X86/throws-cfi-fp.ll        |   7 +-
 llvm/test/CodeGen/X86/throws-cfi-no-fp.ll     |   2 +-
 35 files changed, 1756 insertions(+), 1954 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 3888f141b5370..880a48df56466 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -899,12 +899,14 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
     return 30;
   }
 
-  /// Return true if the greedy register allocator should take
-  /// the lowest-cost choice before using an unused CSR.
-  /// This uses tryAssignCSRFirstTimeMinCost instead of
-  /// tryAssignCSRFirstTime.
-  virtual bool enableCSRFirstTimeMinCost(const MachineFunction &MF) const {
-    return false;
+  /// Return true if the RegAlloc can spill instead of using an unused CSR,
+  /// when the spill costs less than the CSR cost. Return false if a spill
+  /// on this target is not comparable with the CSR cost, for example when
+  /// a spill does not go to the stack.
+  virtual bool
+  shouldSpillInsteadOfFirstCSRUse(const MachineFunction &MF,
+                                  const LiveInterval &VirtReg) const {
+    return true;
   }
 
   /// Returns true if the target requires (and can make use of) the register
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index acce796d5ff65..55c11987c0898 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2471,8 +2471,8 @@ BlockFrequency RAGreedy::calcRematCost(const LiveInterval &VirtReg,
   return HasUse ? RematCost : BlockFrequency::max();
 }
 
-/// Return true if the spiller can rematerialize each use of \p LI.
-/// And it can rematerialize for only one def.
+/// Return true if the spiller can rematerialize each use of LI.
+/// And it can rematerialize for only one def, which does not load.
 static bool isRematerializableFromOriginal(const LiveInterval &LI,
                                            const LiveIntervals &LIS,
                                            const VirtRegMap &VRM,
@@ -2483,17 +2483,18 @@ static bool isRematerializableFromOriginal(const LiveInterval &LI,
   for (const VNInfo *VNI : OrigLI.valnos) {
     if (VNI->isUnused())
       continue;
-    // Maybe more than one value def for VirtReg
+    // The original live range must have only one def.
     if (VNI->isPHIDef() || Def)
       return false;
     Def = LIS.getInstructionFromIndex(VNI->def);
-    if (!Def || !TII.isReMaterializable(*Def))
+    if (!Def || !TII.isReMaterializable(*Def) || Def->mayLoad())
       return false;
   }
   if (!Def)
     return false;
   for (const MachineInstr &MI : MRI.use_nodbg_instructions(LI.reg()))
-    if (!VirtRegAuxInfo::allUsesAvailableAt(Def, LIS.getInstructionIndex(MI),
+    if (!canGuaranteeAssignmentAfterRemat(LI.reg(), MI) ||
+        !VirtRegAuxInfo::allUsesAvailableAt(Def, LIS.getInstructionIndex(MI),
                                             LIS, MRI, TII))
       return false;
   return true;
@@ -2525,6 +2526,10 @@ MCRegister RAGreedy::tryAssignCSRFirstTime(
     uint8_t &CostPerUseLimit, SmallVectorImpl<Register> &NewVRegs) {
   LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
 
+  // For RS_Done, just use the CSR.
+  if (Stage > RS_Spill)
+    return PhysReg;
+
   if (Stage == RS_Spill && VirtReg.isSpillable()) {
     // We choose spill or rematerialize over using the CSR for the first time if
     // the spill cost is lower than CSRCost.
@@ -2539,116 +2544,61 @@ MCRegister RAGreedy::tryAssignCSRFirstTime(
     CostPerUseLimit = 1;
     return MCRegister();
   }
+
+  // The lowest cost of rematerializing or spilling VirtReg.
+  BlockFrequency SpillCost = BlockFrequency::max();
+  if (VirtReg.isSpillable()) {
+    if (isRematerializableFromOriginal(VirtReg, *LIS, *VRM, *MRI, *TII))
+      SpillCost = calcRematCostAtUses(VirtReg, *MRI, *SpillPlacer);
+    if (TRI->shouldSpillInsteadOfFirstCSRUse(*MF, VirtReg))
+      SpillCost = std::min(SpillCost, calcSpillCost(VirtReg));
+  }
+
   if (Stage < RS_Split) {
     // We choose pre-splitting over using the CSR for the first time if
-    // the cost of splitting is lower than CSRCost.
+    // the cost of splitting is lower than CSRCost and the remat/spill cost.
     SA->analyze(&VirtReg);
     unsigned NumCands = 0;
     BlockFrequency BestCost = CSRCost; // Don't modify CSRCost.
     unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, BestCost,
                                                  NumCands, true /*IgnoreCSR*/);
-    if (BestCand == NoCand) {
-      if (calcRematCost(VirtReg, Order) < CSRCost) {
-        CostPerUseLimit = 1;
-        return MCRegister();
-      }
-      // Use the CSR if we can't find a region split below CSRCost.
-      return PhysReg;
+    if (BestCand != NoCand && BestCost <= SpillCost) {
+      // Perform the actual pre-splitting.
+      doRegionSplit(VirtReg, BestCand, false /*HasCompact*/, NewVRegs);
+      return MCRegister();
     }
-
-    // Perform the actual pre-splitting.
-    doRegionSplit(VirtReg, BestCand, false/*HasCompact*/, NewVRegs);
-    return MCRegister();
   }
   if (calcRematCost(VirtReg, Order) < CSRCost) {
     CostPerUseLimit = 1;
     return MCRegister();
   }
-  return PhysReg;
-}
-
-// Like tryAssignCSRFirstTime before assign the unused CSR to VirtReg, try
-// other choices. There are four results:
-//   1. Pre-split VirtReg
-//   2. Remat/Spill VirtReg
-//   3. Evict rematerializable live ranges if that reg is not unused CSR.
-//   4. Use the CSR for this VirtReg
-// The goal is using the less cost register.
-MCRegister RAGreedy::tryAssignCSRFirstTimeMinCost(
-    const LiveInterval &VirtReg, AllocationOrder &Order, MCRegister PhysReg,
-    uint8_t &CostPerUseLimit, SmallVectorImpl<Register> &NewVRegs) {
-  // Compute Remat cost.
-  std::optional<BlockFrequency> RematCost;
-  if (VirtReg.isSpillable() &&
-      isRematerializableFromOriginal(VirtReg, *LIS, *VRM, *MRI, *TII))
-    RematCost = calcRematCostAtUses(VirtReg, *MRI, *SpillPlacer);
-
-  std::optional<BlockFrequency> SpillRematCost;
-  if (VirtReg.isSpillable()) {
-    SpillRematCost = calcSpillCost(VirtReg);
-    if (RematCost)
-      SpillRematCost = std::min(*SpillRematCost, *RematCost);
-  }
-
-  LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
-
-  if (Stage == RS_Spill && VirtReg.isSpillable()) {
-    // We choose spill or rematerialize over using the CSR for the first time if
-    // the spill cost is lower than CSRCost.
-    if (*SpillRematCost >= CSRCost)
-      return PhysReg;
-
-    // We are going to spill or rematerialize, set CostPerUseLimit to 1 to make
-    // sure that we will not use a callee-saved register in tryEvict.
+  // Evict rematerializable live ranges from a register that is not an unused
+  // CSR, if that costs less than CSRCost and the spill cost.
+  if (MCRegister Reg = tryEvictRematForCSR(VirtReg, Order, NewVRegs,
+                                           std::min(CSRCost, SpillCost)))
+    return Reg;
+  // We choose spilling or rematerialization over using the CSR for the first
+  // time if it costs less than CSRCost.
+  if (SpillCost < CSRCost) {
+    // Set CostPerUseLimit to 1 to make sure that we will not use a
+    // callee-saved register in tryEvict.
+    ExtraInfo->setStage(VirtReg, RS_Spill);
     CostPerUseLimit = 1;
     return MCRegister();
   }
-
-  if (Stage < RS_Split) {
-    SA->analyze(&VirtReg);
-    // A pre-split must cost less than CSRCost.
-    // CSRCost is the cost for place the CSR save/restore. And it be set by
-    // target.
-    BlockFrequency SplitCost = CSRCost;
-    unsigned NumCands = 0;
-    unsigned BestCand = calculateRegionSplitCost(VirtReg, Order, SplitCost,
-                                                 NumCands, true /*IgnoreCSR*/);
-
-    bool SpillRematCostsLess = SpillRematCost && *SpillRematCost < SplitCost;
-    if (BestCand != NoCand && !SpillRematCostsLess) {
-      doRegionSplit(VirtReg, BestCand, false /*HasCompact*/, NewVRegs);
-      return MCRegister();
-    }
-  }
-
-  bool SpillCostsLess = SpillRematCost && *SpillRematCost < CSRCost;
-  BlockFrequency EvictLimit = SpillCostsLess ? *SpillRematCost : CSRCost;
-  // Use Evict Reg
-  if (MCRegister Reg =
-          tryEvictRematForCSR(VirtReg, Order, NewVRegs, EvictLimit))
-    return Reg;
-
-  // Use CSR
-  if (!SpillCostsLess)
-    return PhysReg;
-
-  // Use Spill/Remat
-  // We are going to spill, set CostPerUseLimit to 1 to make sure that
-  // we will not use a callee-saved register in tryEvict.
-  ExtraInfo->setStage(VirtReg, RS_Spill);
-  CostPerUseLimit = 1;
-  return MCRegister();
+  // Use the CSR if no other choice costs less than CSRCost.
+  return PhysReg;
 }
 
 /// Return the cost of evicting and rematerializing all the live ranges that
-/// interfere with \p VirtReg in \p PhysReg.
-/// Return std::nullopt when
-///  1. has lower Cascade than \p Cascade.
-///  2. can't be spilled.
+/// interfere with VirtReg in PhysReg.
+/// Return BlockFrequency::max() when one of these live ranges
+///  1. has a Cascade that is not lower than Cascade,
+///  2. can't be spilled, or
 ///  3. can't be rematerialized.
-std::optional<BlockFrequency>
-RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
-                             unsigned Cascade) const {
+BlockFrequency RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg,
+                                            MCRegister PhysReg,
+                                            unsigned Cascade) const {
   BlockFrequency Cost(0);
   SmallPtrSet<const LiveInterval *, 4> Seen;
   for (MCRegUnit Unit : TRI->regunits(PhysReg)) {
@@ -2659,16 +2609,16 @@ RAGreedy::calcEvictRematCost(const LiveInterval &VirtReg, MCRegister PhysReg,
       if (ExtraInfo->getCascade(Intf->reg()) >= Cascade ||
           !Intf->isSpillable() ||
           !isRematerializableFromOriginal(*Intf, *LIS, *VRM, *MRI, *TII))
-        return std::nullopt;
+        return BlockFrequency::max();
       Cost += calcRematCostAtUses(*Intf, *MRI, *SpillPlacer);
     }
   }
   return Cost;
 }
 
-/// Before \p VirtReg uses a CSR for the first time, look for a register that
+/// Before VirtReg uses a CSR for the first time, look for a register that
 /// is not an unused CSR and that only rematerializable live ranges occupy.
-/// If rematerializing them costs less than \p MaxCost, evict them and
+/// If rematerializing them costs less than MaxCost, evict them and
 /// return the register.
 MCRegister RAGreedy::tryEvictRematForCSR(const LiveInterval &VirtReg,
                                          AllocationOrder &Order,
@@ -2682,10 +2632,9 @@ MCRegister RAGreedy::tryEvictRematForCSR(const LiveInterval &VirtReg,
         Matrix->checkInterference(VirtReg, PhysReg) !=
             LiveRegMatrix::IK_VirtReg)
       continue;
-    std::optional<BlockFrequency> Cost =
-        calcEvictRematCost(VirtReg, PhysReg, Cascade);
-    if (Cost && *Cost < BestCost) {
-      BestCost = *Cost;
+    BlockFrequency Cost = calcEvictRematCost(VirtReg, PhysReg, Cascade);
+    if (Cost < BestCost) {
+      BestCost = Cost;
       BestReg = PhysReg;
     }
   }
@@ -2963,12 +2912,8 @@ MCRegister RAGreedy::selectOrSplitImpl(const LiveInterval &VirtReg,
     if (CSRCost.getFrequency() &&
         EvictAdvisor->getUnusedCalleeSavedReg(VirtReg, PhysReg) &&
         NewVRegs.empty()) {
-      MCRegister CSRReg =
-          TRI->enableCSRFirstTimeMinCost(*MF)
-              ? tryAssignCSRFirstTimeMinCost(VirtReg, Order, PhysReg,
-                                             CostPerUseLimit, NewVRegs)
-              : tryAssignCSRFirstTime(VirtReg, Order, PhysReg, CostPerUseLimit,
-                                      NewVRegs);
+      MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
+                                                CostPerUseLimit, NewVRegs);
       if (CSRReg || !NewVRegs.empty())
         // Return now if we decide to use a CSR or create new vregs due to
         // pre-splitting.
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 4335343942320..16273854eaaf6 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -365,17 +365,8 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
   /// rematerialization is not possible.
   BlockFrequency calcRematCost(const LiveInterval &VirtReg,
                                AllocationOrder &Order) const;
-  /// Like tryAssignCSRFirstTime, but take the choice with the lowest cost of:
-  /// a pre-split, spilling or rematerializing VirtReg, evicting
-  /// rematerializable live ranges, and the first use of the CSR.
-  MCRegister tryAssignCSRFirstTimeMinCost(const LiveInterval &VirtReg,
-                                          AllocationOrder &Order,
-                                          MCRegister PhysReg,
-                                          uint8_t &CostPerUseLimit,
-                                          SmallVectorImpl<Register> &NewVRegs);
-  std::optional<BlockFrequency> calcEvictRematCost(const LiveInterval &VirtReg,
-                                                   MCRegister PhysReg,
-                                                   unsigned Cascade) const;
+  BlockFrequency calcEvictRematCost(const LiveInterval &VirtReg,
+                                    MCRegister PhysReg, unsigned Cascade) const;
   MCRegister tryEvictRematForCSR(const LiveInterval &VirtReg,
                                  AllocationOrder &Order,
                                  SmallVectorImpl<Register> &NewVRegs,
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 4cacba0ed2592..a1362f34848e2 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -128,6 +128,15 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
     return 27;
   }
 
+  bool
+  shouldSpillInsteadOfFirstCSRUse(const MachineFunction &MF,
+                                  const LiveInterval &VirtReg) const override {
+    // An SGPR spill goes to a VGPR lane, like a CSR SGPR save, but at each use,
+    // and it needs one more WWM VGPR. getCSRFirstUseCost also models occupancy,
+    // so it is not comparable with the spill cost.
+    return false;
+  }
+
   bool shouldCompactSlotIndexes(const MachineFunction &MF) const override {
     return true;
   }
diff --git a/llvm/test/CodeGen/AArch64/call-rv-marker.ll b/llvm/test/CodeGen/AArch64/call-rv-marker.ll
index a2a9138f57f9a..682f0435ab0dd 100644
--- a/llvm/test/CodeGen/AArch64/call-rv-marker.ll
+++ b/llvm/test/CodeGen/AArch64/call-rv-marker.ll
@@ -89,9 +89,10 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
 ; CHECK-SD-NEXT:    .cfi_personality 155, ___gxx_personality_v0
 ; CHECK-SD-NEXT:    .cfi_lsda 16, Lexception0
 ; CHECK-SD-NEXT:  ; %bb.0: ; %entry
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-SD-NEXT:    sub sp, sp, #48
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #16] ; 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x29, x30, [sp, #32] ; 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-SD-NEXT:    .cfi_offset w30, -8
 ; CHECK-SD-NEXT:    .cfi_offset w29, -16
 ; CHECK-SD-NEXT:    .cfi_offset w19, -24
@@ -104,16 +105,17 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
 ; CHECK-SD-NEXT:    bl _objc_object
 ; CHECK-SD-NEXT:  Ltmp1: ; EH_LABEL
 ; CHECK-SD-NEXT:  ; %bb.1: ; %invoke.cont
-; CHECK-SD-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x29, x30, [sp, #32] ; 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #16] ; 16-byte Folded Reload
+; CHECK-SD-NEXT:    add sp, sp, #48
 ; CHECK-SD-NEXT:    b _objc_release
 ; CHECK-SD-NEXT:  LBB3_2: ; %lpad
 ; CHECK-SD-NEXT:  Ltmp2: ; EH_LABEL
-; CHECK-SD-NEXT:    mov x20, x0
+; CHECK-SD-NEXT:    str x0, [sp, #8] ; 8-byte Spill
 ; CHECK-SD-NEXT:    mov x0, x19
 ; CHECK-SD-NEXT:    bl _objc_release
-; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    ldr x0, [sp, #8] ; 8-byte Reload
 ; CHECK-SD-NEXT:    bl __Unwind_Resume
 ; CHECK-SD-NEXT:  Lfunc_end0:
 ; CHECK-SD-NEXT:    .cfi_endproc
@@ -147,9 +149,10 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
 ; CHECK-GI-NEXT:    .cfi_personality 155, ___gxx_personality_v0
 ; CHECK-GI-NEXT:    .cfi_lsda 16, Lexception0
 ; CHECK-GI-NEXT:  ; %bb.0: ; %entry
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-GI-NEXT:    sub sp, sp, #48
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #16] ; 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x29, x30, [sp, #32] ; 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-GI-NEXT:    .cfi_offset w30, -8
 ; CHECK-GI-NEXT:    .cfi_offset w29, -16
 ; CHECK-GI-NEXT:    .cfi_offset w19, -24
@@ -167,19 +170,20 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 {
 ; CHECK-GI-NEXT:    mov x0, x19
 ; CHECK-GI-NEXT:  Lloh1:
 ; CHECK-GI-NEXT:    ldr x1, [x1, _objc_release at GOTPAGEOFF]
-; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x29, x30, [sp, #32] ; 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] ; 16-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #48
 ; CHECK-GI-NEXT:    br x1
 ; CHECK-GI-NEXT:  LBB3_2: ; %lpad
 ; CHECK-GI-NEXT:  Ltmp2: ; EH_LABEL
 ; CHECK-GI-NEXT:  Lloh2:
 ; CHECK-GI-NEXT:    adrp x8, _objc_release at GOTPAGE
-; CHECK-GI-NEXT:    mov x20, x0
-; CHECK-GI-NEXT:    mov x0, x19
 ; CHECK-GI-NEXT:  Lloh3:
 ; CHECK-GI-NEXT:    ldr x8, [x8, _objc_release at GOTPAGEOFF]
+; CHECK-GI-NEXT:    str x0, [sp, #8] ; 8-byte Spill
+; CHECK-GI-NEXT:    mov x0, x19
 ; CHECK-GI-NEXT:    blr x8
-; CHECK-GI-NEXT:    mov x0, x20
+; CHECK-GI-NEXT:    ldr x0, [sp, #8] ; 8-byte Reload
 ; CHECK-GI-NEXT:    bl __Unwind_Resume
 ; CHECK-GI-NEXT:    .loh AdrpLdrGot Lloh0, Lloh1
 ; CHECK-GI-NEXT:    .loh AdrpLdrGot Lloh2, Lloh3
@@ -260,18 +264,17 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 {
 ; CHECK-SD-NEXT:    ret
 ; CHECK-SD-NEXT:  LBB4_3: ; %lpad1
 ; CHECK-SD-NEXT:  Ltmp8: ; EH_LABEL
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x0, x1
+; CHECK-SD-NEXT:    str x0, [sp] ; 8-byte Spill
+; CHECK-SD-NEXT:    mov x0, x19
 ; CHECK-SD-NEXT:    bl _objc_release
 ; CHECK-SD-NEXT:    b LBB4_5
 ; CHECK-SD-NEXT:  LBB4_4: ; %lpad
 ; CHECK-SD-NEXT:  Ltmp5: ; EH_LABEL
-; CHECK-SD-NEXT:    mov x19, x0
+; CHECK-SD-NEXT:    str x0, [sp] ; 8-byte Spill
 ; CHECK-SD-NEXT:  LBB4_5: ; %ehcleanup
 ; CHECK-SD-NEXT:    add x0, sp, #15
 ; CHECK-SD-NEXT:    bl __ZN1SD1Ev
-; CHECK-SD-NEXT:    mov x0, x19
+; CHECK-SD-NEXT:    ldr x0, [sp] ; 8-byte Reload
 ; CHECK-SD-NEXT:    bl __Unwind_Resume
 ; CHECK-SD-NEXT:  Lfunc_end1:
 ; CHECK-SD-NEXT:    .cfi_endproc
@@ -340,19 +343,19 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 {
 ; CHECK-GI-NEXT:  Ltmp8: ; EH_LABEL
 ; CHECK-GI-NEXT:  Lloh6:
 ; CHECK-GI-NEXT:    adrp x8, _objc_release at GOTPAGE
-; CHECK-GI-NEXT:    mov x20, x0
-; CHECK-GI-NEXT:    mov x0, x19
 ; CHECK-GI-NEXT:  Lloh7:
 ; CHECK-GI-NEXT:    ldr x8, [x8, _objc_release at GOTPAGEOFF]
+; CHECK-GI-NEXT:    str x0, [sp] ; 8-byte Spill
+; CHECK-GI-NEXT:    mov x0, x19
 ; CHECK-GI-NEXT:    blr x8
 ; CHECK-GI-NEXT:    b LBB4_5
 ; CHECK-GI-NEXT:  LBB4_4: ; %lpad
 ; CHECK-GI-NEXT:  Ltmp5: ; EH_LABEL
-; CHECK-GI-NEXT:    mov x20, x0
+; CHECK-GI-NEXT:    str x0, [sp] ; 8-byte Spill
 ; CHECK-GI-NEXT:  LBB4_5: ; %ehcleanup
 ; CHECK-GI-NEXT:    add x0, sp, #15
 ; CHECK-GI-NEXT:    bl __ZN1SD1Ev
-; CHECK-GI-NEXT:    mov x0, x20
+; CHECK-GI-NEXT:    ldr x0, [sp] ; 8-byte Reload
 ; CHECK-GI-NEXT:    bl __Unwind_Resume
 ; CHECK-GI-NEXT:    .loh AdrpLdrGot Lloh4, Lloh5
 ; CHECK-GI-NEXT:    .loh AdrpLdrGot Lloh6, Lloh7
diff --git a/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll b/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
index c353a3e6496a8..a48c3e5e9b5c2 100644
--- a/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
+++ b/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
@@ -29,14 +29,12 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
 ; CHECK-NEXT:    .cfi_personality 156, DW.ref.__gxx_personality_v0
 ; CHECK-NEXT:    .cfi_lsda 28, .Lexception0
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #16] // 8-byte Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x29, sp
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa w29, 32
-; CHECK-NEXT:    .cfi_offset w19, -16
-; CHECK-NEXT:    .cfi_offset w30, -24
-; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    rdsvl x8, #1
 ; CHECK-NEXT:    mov x9, sp
 ; CHECK-NEXT:    msub x9, x8, x8, x9
@@ -45,8 +43,7 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
 ; CHECK-NEXT:    tbnz w0, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %return_normally
 ; CHECK-NEXT:    mov sp, x29
-; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Reload
-; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    b shared_za_call
 ; CHECK-NEXT:  .LBB0_2: // %throw_exception
 ; CHECK-NEXT:    sub x8, x29, #16
@@ -73,7 +70,7 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
 ; CHECK-NEXT:  // %bb.5: // %throw_fail
 ; CHECK-NEXT:  .LBB0_6: // %unwind_dtors
 ; CHECK-NEXT:  .Ltmp2: // EH_LABEL
-; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    stur x0, [x29, #-24] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart za
 ; CHECK-NEXT:    mrs x8, TPIDR2_EL0
 ; CHECK-NEXT:    sub x0, x29, #16
@@ -84,8 +81,8 @@ define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_pe
 ; CHECK-NEXT:    msr TPIDR2_EL0, xzr
 ; CHECK-NEXT:    bl shared_za_call
 ; CHECK-NEXT:    sub x8, x29, #16
-; CHECK-NEXT:    mov x0, x19
 ; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    ldur x0, [x29, #-24] // 8-byte Folded Reload
 ; CHECK-NEXT:    bl _Unwind_Resume
   br i1 %fail, label %throw_exception, label %return_normally
 
@@ -306,13 +303,13 @@ define void @try_catch_shared_zt0_callee() "aarch64_inout_zt0" personality ptr @
 ; CHECK-NEXT:    .cfi_personality 156, DW.ref.__gxx_personality_v0
 ; CHECK-NEXT:    .cfi_lsda 28, .Lexception3
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #80
-; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    sub sp, sp, #96
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:  .Ltmp9: // EH_LABEL
-; CHECK-NEXT:    mov x19, sp
+; CHECK-NEXT:    add x19, sp, #16
 ; CHECK-NEXT:    str zt0, [x19]
 ; CHECK-NEXT:    smstop za
 ; CHECK-NEXT:    bl may_throw
@@ -320,20 +317,18 @@ define void @try_catch_shared_zt0_callee() "aarch64_inout_zt0" personality ptr @
 ; CHECK-NEXT:    smstart za
 ; CHECK-NEXT:    ldr zt0, [x19]
 ; CHECK-NEXT:  // %bb.1: // %return_normally
-; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB3_2: // %unwind_dtors
 ; CHECK-NEXT:  .Ltmp11: // EH_LABEL
-; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    str x0, [sp, #8] // 8-byte Spill
 ; CHECK-NEXT:    smstart za
-; CHECK-NEXT:    ldr zt0, [x8]
+; CHECK-NEXT:    ldr zt0, [x19]
 ; CHECK-NEXT:    bl shared_zt0_call
-; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    str zt0, [x8]
+; CHECK-NEXT:    str zt0, [x19]
 ; CHECK-NEXT:    smstop za
-; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    ldr x0, [sp, #8] // 8-byte Reload
 ; CHECK-NEXT:    bl _Unwind_Resume
   invoke void @may_throw()
           to label %return_normally unwind label %unwind_dtors
@@ -601,7 +596,7 @@ define void @try_catch_shared_za_callee_zt0_saved(ptr %callee) "aarch64_inout_za
 ; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x29, sp
-; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    .cfi_def_cfa w29, 32
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
@@ -636,7 +631,7 @@ define void @try_catch_shared_za_callee_zt0_saved(ptr %callee) "aarch64_inout_za
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB8_4: // %unwind_dtors
 ; CHECK-NEXT:  .Ltmp26: // EH_LABEL
-; CHECK-NEXT:    mov x20, x0
+; CHECK-NEXT:    stur x0, [x29, #-88] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart za
 ; CHECK-NEXT:    mrs x8, TPIDR2_EL0
 ; CHECK-NEXT:    sub x0, x29, #80
@@ -647,8 +642,8 @@ define void @try_catch_shared_za_callee_zt0_saved(ptr %callee) "aarch64_inout_za
 ; CHECK-NEXT:    msr TPIDR2_EL0, xzr
 ; CHECK-NEXT:    blr x19
 ; CHECK-NEXT:    sub x8, x29, #80
-; CHECK-NEXT:    mov x0, x20
 ; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    ldur x0, [x29, #-88] // 8-byte Folded Reload
 ; CHECK-NEXT:    bl _Unwind_Resume
   invoke void @may_throw()
           to label %return_normally unwind label %unwind_dtors
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 5da163b24d948..add1020821e6f 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -12352,22 +12352,16 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-LABEL: v_fadd_v32bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
 ; GFX950-NEXT:    scratch_load_dword v31, off, s32
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v24
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v8
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT:    v_add_f32_e32 v8, v8, v24
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_add_f32_e32 v7, v7, v23
-; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT:    v_add_f32_e32 v24, v8, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX950-NEXT:    v_and_b32_e32 v33, 0xffff0000, v30
 ; GFX950-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
@@ -12405,7 +12399,6 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_add_f32_e32 v26, v52, v51
 ; GFX950-NEXT:    v_add_f32_e32 v9, v9, v25
 ; GFX950-NEXT:    v_add_f32_e32 v25, v54, v53
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v8, v25
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, v26
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, v27
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, v28
@@ -12413,50 +12406,55 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, v30
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v14, v33
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
-; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT:    v_add_f32_e32 v24, v32, v24
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT:    v_add_f32_e32 v32, v32, v8
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT:    v_add_f32_e32 v15, v15, v31
+; GFX950-NEXT:    v_add_f32_e32 v7, v7, v23
+; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_add_f32_e32 v23, v32, v23
+; GFX950-NEXT:    v_add_f32_e32 v23, v31, v23
 ; GFX950-NEXT:    v_add_f32_e32 v6, v6, v22
 ; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_add_f32_e32 v22, v32, v22
+; GFX950-NEXT:    v_add_f32_e32 v22, v31, v22
 ; GFX950-NEXT:    v_add_f32_e32 v5, v5, v21
 ; GFX950-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_add_f32_e32 v21, v32, v21
+; GFX950-NEXT:    v_add_f32_e32 v21, v31, v21
 ; GFX950-NEXT:    v_add_f32_e32 v4, v4, v20
 ; GFX950-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_add_f32_e32 v20, v32, v20
+; GFX950-NEXT:    v_add_f32_e32 v20, v31, v20
 ; GFX950-NEXT:    v_add_f32_e32 v3, v3, v19
 ; GFX950-NEXT:    v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_add_f32_e32 v19, v32, v19
+; GFX950-NEXT:    v_add_f32_e32 v19, v31, v19
 ; GFX950-NEXT:    v_add_f32_e32 v2, v2, v18
 ; GFX950-NEXT:    v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT:    v_add_f32_e32 v18, v32, v18
+; GFX950-NEXT:    v_add_f32_e32 v18, v31, v18
 ; GFX950-NEXT:    v_add_f32_e32 v1, v1, v17
 ; GFX950-NEXT:    v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_add_f32_e32 v15, v15, v31
-; GFX950-NEXT:    v_add_f32_e32 v31, v40, v55
-; GFX950-NEXT:    v_add_f32_e32 v17, v32, v17
+; GFX950-NEXT:    v_add_f32_e32 v8, v55, v8
+; GFX950-NEXT:    v_add_f32_e32 v17, v31, v17
 ; GFX950-NEXT:    v_add_f32_e32 v0, v0, v16
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v17
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, v18
@@ -12465,9 +12463,9 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, v21
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, v22
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v32
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fadd_v32bf16:
@@ -17521,22 +17519,16 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-LABEL: v_fmul_v32bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
 ; GFX950-NEXT:    scratch_load_dword v31, off, s32
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v24
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v8
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT:    v_mul_f32_e32 v8, v8, v24
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_mul_f32_e32 v7, v7, v23
-; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT:    v_mul_f32_e32 v24, v8, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX950-NEXT:    v_and_b32_e32 v33, 0xffff0000, v30
 ; GFX950-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
@@ -17574,7 +17566,6 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_mul_f32_e32 v26, v52, v51
 ; GFX950-NEXT:    v_mul_f32_e32 v9, v9, v25
 ; GFX950-NEXT:    v_mul_f32_e32 v25, v54, v53
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v8, v25
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, v26
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, v27
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, v28
@@ -17582,50 +17573,55 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, v30
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v14, v33
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
-; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT:    v_mul_f32_e32 v24, v32, v24
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT:    v_mul_f32_e32 v32, v32, v8
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT:    v_mul_f32_e32 v15, v15, v31
+; GFX950-NEXT:    v_mul_f32_e32 v7, v7, v23
+; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_mul_f32_e32 v23, v32, v23
+; GFX950-NEXT:    v_mul_f32_e32 v23, v31, v23
 ; GFX950-NEXT:    v_mul_f32_e32 v6, v6, v22
 ; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_mul_f32_e32 v22, v32, v22
+; GFX950-NEXT:    v_mul_f32_e32 v22, v31, v22
 ; GFX950-NEXT:    v_mul_f32_e32 v5, v5, v21
 ; GFX950-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_mul_f32_e32 v21, v32, v21
+; GFX950-NEXT:    v_mul_f32_e32 v21, v31, v21
 ; GFX950-NEXT:    v_mul_f32_e32 v4, v4, v20
 ; GFX950-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_mul_f32_e32 v20, v32, v20
+; GFX950-NEXT:    v_mul_f32_e32 v20, v31, v20
 ; GFX950-NEXT:    v_mul_f32_e32 v3, v3, v19
 ; GFX950-NEXT:    v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_mul_f32_e32 v19, v32, v19
+; GFX950-NEXT:    v_mul_f32_e32 v19, v31, v19
 ; GFX950-NEXT:    v_mul_f32_e32 v2, v2, v18
 ; GFX950-NEXT:    v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT:    v_mul_f32_e32 v18, v32, v18
+; GFX950-NEXT:    v_mul_f32_e32 v18, v31, v18
 ; GFX950-NEXT:    v_mul_f32_e32 v1, v1, v17
 ; GFX950-NEXT:    v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_mul_f32_e32 v15, v15, v31
-; GFX950-NEXT:    v_mul_f32_e32 v31, v40, v55
-; GFX950-NEXT:    v_mul_f32_e32 v17, v32, v17
+; GFX950-NEXT:    v_mul_f32_e32 v8, v55, v8
+; GFX950-NEXT:    v_mul_f32_e32 v17, v31, v17
 ; GFX950-NEXT:    v_mul_f32_e32 v0, v0, v16
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v17
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, v18
@@ -17634,9 +17630,9 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, v21
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, v22
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v32
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fmul_v32bf16:
@@ -22516,22 +22512,16 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-LABEL: v_minnum_v32bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
 ; GFX950-NEXT:    scratch_load_dword v31, off, s32
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v24
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v8
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT:    v_min_f32_e32 v8, v8, v24
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_min_f32_e32 v7, v7, v23
-; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT:    v_min_f32_e32 v24, v8, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX950-NEXT:    v_and_b32_e32 v33, 0xffff0000, v30
 ; GFX950-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
@@ -22569,7 +22559,6 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_min_f32_e32 v26, v52, v51
 ; GFX950-NEXT:    v_min_f32_e32 v9, v9, v25
 ; GFX950-NEXT:    v_min_f32_e32 v25, v54, v53
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v8, v25
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, v26
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, v27
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, v28
@@ -22577,50 +22566,55 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, v30
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v14, v33
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
-; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT:    v_min_f32_e32 v24, v32, v24
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT:    v_min_f32_e32 v32, v32, v8
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT:    v_min_f32_e32 v15, v15, v31
+; GFX950-NEXT:    v_min_f32_e32 v7, v7, v23
+; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_min_f32_e32 v23, v32, v23
+; GFX950-NEXT:    v_min_f32_e32 v23, v31, v23
 ; GFX950-NEXT:    v_min_f32_e32 v6, v6, v22
 ; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_min_f32_e32 v22, v32, v22
+; GFX950-NEXT:    v_min_f32_e32 v22, v31, v22
 ; GFX950-NEXT:    v_min_f32_e32 v5, v5, v21
 ; GFX950-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_min_f32_e32 v21, v32, v21
+; GFX950-NEXT:    v_min_f32_e32 v21, v31, v21
 ; GFX950-NEXT:    v_min_f32_e32 v4, v4, v20
 ; GFX950-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_min_f32_e32 v20, v32, v20
+; GFX950-NEXT:    v_min_f32_e32 v20, v31, v20
 ; GFX950-NEXT:    v_min_f32_e32 v3, v3, v19
 ; GFX950-NEXT:    v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_min_f32_e32 v19, v32, v19
+; GFX950-NEXT:    v_min_f32_e32 v19, v31, v19
 ; GFX950-NEXT:    v_min_f32_e32 v2, v2, v18
 ; GFX950-NEXT:    v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT:    v_min_f32_e32 v18, v32, v18
+; GFX950-NEXT:    v_min_f32_e32 v18, v31, v18
 ; GFX950-NEXT:    v_min_f32_e32 v1, v1, v17
 ; GFX950-NEXT:    v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_min_f32_e32 v15, v15, v31
-; GFX950-NEXT:    v_min_f32_e32 v31, v40, v55
-; GFX950-NEXT:    v_min_f32_e32 v17, v32, v17
+; GFX950-NEXT:    v_min_f32_e32 v8, v55, v8
+; GFX950-NEXT:    v_min_f32_e32 v17, v31, v17
 ; GFX950-NEXT:    v_min_f32_e32 v0, v0, v16
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v17
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, v18
@@ -22629,9 +22623,9 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, v21
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, v22
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v32
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minnum_v32bf16:
@@ -26943,22 +26937,16 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-LABEL: v_maxnum_v32bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_accvgpr_write_b32 a0, v40 ; Reload Reuse
 ; GFX950-NEXT:    scratch_load_dword v31, off, s32
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v24
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v8
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v15
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v23
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v7
-; GFX950-NEXT:    v_max_f32_e32 v8, v8, v24
-; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
-; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX950-NEXT:    v_max_f32_e32 v7, v7, v23
-; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
-; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
+; GFX950-NEXT:    v_max_f32_e32 v24, v8, v24
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v7
+; GFX950-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX950-NEXT:    v_and_b32_e32 v33, 0xffff0000, v30
 ; GFX950-NEXT:    v_and_b32_e32 v34, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
@@ -26996,7 +26984,6 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_max_f32_e32 v26, v52, v51
 ; GFX950-NEXT:    v_max_f32_e32 v9, v9, v25
 ; GFX950-NEXT:    v_max_f32_e32 v25, v54, v53
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v8, v25
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v9, v26
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v10, v27
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v11, v28
@@ -27004,50 +26991,55 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v13, v30
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v14, v33
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
-; GFX950-NEXT:    v_and_b32_e32 v24, 0xffff0000, v31
-; GFX950-NEXT:    v_max_f32_e32 v24, v32, v24
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v6
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
+; GFX950-NEXT:    v_max_f32_e32 v32, v32, v8
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v23
+; GFX950-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX950-NEXT:    v_max_f32_e32 v15, v15, v31
+; GFX950-NEXT:    v_max_f32_e32 v7, v7, v23
+; GFX950-NEXT:    v_and_b32_e32 v23, 0xffff0000, v22
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v6
+; GFX950-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_max_f32_e32 v23, v32, v23
+; GFX950-NEXT:    v_max_f32_e32 v23, v31, v23
 ; GFX950-NEXT:    v_max_f32_e32 v6, v6, v22
 ; GFX950-NEXT:    v_and_b32_e32 v22, 0xffff0000, v21
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v5
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX950-NEXT:    v_max_f32_e32 v22, v32, v22
+; GFX950-NEXT:    v_max_f32_e32 v22, v31, v22
 ; GFX950-NEXT:    v_max_f32_e32 v5, v5, v21
 ; GFX950-NEXT:    v_and_b32_e32 v21, 0xffff0000, v20
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v4
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v20, 16, v20
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX950-NEXT:    v_max_f32_e32 v21, v32, v21
+; GFX950-NEXT:    v_max_f32_e32 v21, v31, v21
 ; GFX950-NEXT:    v_max_f32_e32 v4, v4, v20
 ; GFX950-NEXT:    v_and_b32_e32 v20, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v3
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_max_f32_e32 v20, v32, v20
+; GFX950-NEXT:    v_max_f32_e32 v20, v31, v20
 ; GFX950-NEXT:    v_max_f32_e32 v3, v3, v19
 ; GFX950-NEXT:    v_and_b32_e32 v19, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v2
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT:    v_max_f32_e32 v19, v32, v19
+; GFX950-NEXT:    v_max_f32_e32 v19, v31, v19
 ; GFX950-NEXT:    v_max_f32_e32 v2, v2, v18
 ; GFX950-NEXT:    v_and_b32_e32 v18, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v1
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
-; GFX950-NEXT:    v_max_f32_e32 v18, v32, v18
+; GFX950-NEXT:    v_max_f32_e32 v18, v31, v18
 ; GFX950-NEXT:    v_max_f32_e32 v1, v1, v17
 ; GFX950-NEXT:    v_and_b32_e32 v17, 0xffff0000, v16
-; GFX950-NEXT:    v_and_b32_e32 v32, 0xffff0000, v0
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_max_f32_e32 v15, v15, v31
-; GFX950-NEXT:    v_max_f32_e32 v31, v40, v55
-; GFX950-NEXT:    v_max_f32_e32 v17, v32, v17
+; GFX950-NEXT:    v_max_f32_e32 v8, v55, v8
+; GFX950-NEXT:    v_max_f32_e32 v17, v31, v17
 ; GFX950-NEXT:    v_max_f32_e32 v0, v0, v16
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v17
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, v18
@@ -27056,9 +27048,9 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v4, v4, v21
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v5, v5, v22
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, v23
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v31
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v24
-; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v7, v7, v8
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v24, v25
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v15, v32
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maxnum_v32bf16:
@@ -48752,7 +48744,6 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
 ; GFX950-NEXT:    v_accvgpr_write_b32 a8, v56 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_write_b32 a9, v57 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX950-NEXT:    v_accvgpr_write_b32 a11, v59 ; Reload Reuse
 ; GFX950-NEXT:    scratch_load_dword v31, off, s32 offset:64
 ; GFX950-NEXT:    scratch_load_dword v32, off, s32
 ; GFX950-NEXT:    scratch_load_dword v33, off, s32 offset:60
@@ -48768,45 +48759,47 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
 ; GFX950-NEXT:    scratch_load_dword v51, off, s32 offset:20
 ; GFX950-NEXT:    scratch_load_dword v52, off, s32 offset:16
 ; GFX950-NEXT:    scratch_load_dword v53, off, s32 offset:12
-; GFX950-NEXT:    scratch_load_dword v54, off, s32 offset:8
+; GFX950-NEXT:    scratch_load_dword v58, off, s32 offset:8
 ; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v15
-; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v29
-; GFX950-NEXT:    v_and_b32_e32 v43, 0xffff0000, v13
-; GFX950-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
-; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX950-NEXT:    v_and_b32_e32 v44, 0xffff0000, v28
 ; GFX950-NEXT:    v_and_b32_e32 v45, 0xffff0000, v12
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
-; GFX950-NEXT:    v_and_b32_e32 v47, 0xffff0000, v11
+; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v29
+; GFX950-NEXT:    v_and_b32_e32 v43, 0xffff0000, v13
+; GFX950-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; GFX950-NEXT:    v_and_b32_e32 v57, 0xffff0000, v11
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v30
 ; GFX950-NEXT:    v_and_b32_e32 v41, 0xffff0000, v14
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX950-NEXT:    v_and_b32_e32 v46, 0xffff0000, v27
 ; GFX950-NEXT:    s_waitcnt vmcnt(15)
-; GFX950-NEXT:    v_and_b32_e32 v56, 0xffff0000, v31
+; GFX950-NEXT:    v_and_b32_e32 v46, 0xffff0000, v31
 ; GFX950-NEXT:    s_waitcnt vmcnt(14)
-; GFX950-NEXT:    v_and_b32_e32 v57, 0xffff0000, v32
-; GFX950-NEXT:    v_fmac_f32_e32 v56, v55, v57
+; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v32
+; GFX950-NEXT:    v_fmac_f32_e32 v46, v55, v54
 ; GFX950-NEXT:    s_waitcnt vmcnt(12)
-; GFX950-NEXT:    v_and_b32_e32 v59, 0xffff0000, v34
-; GFX950-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
+; GFX950-NEXT:    v_and_b32_e32 v56, 0xffff0000, v34
 ; GFX950-NEXT:    s_waitcnt vmcnt(11)
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v35
+; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v35
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v35, 16, v35
-; GFX950-NEXT:    v_fmac_f32_e32 v34, v13, v29
-; GFX950-NEXT:    v_fmac_f32_e32 v35, v12, v28
+; GFX950-NEXT:    v_lshlrev_b32_e32 v34, 16, v34
 ; GFX950-NEXT:    s_waitcnt vmcnt(10)
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v36
+; GFX950-NEXT:    v_fmac_f32_e32 v35, v12, v28
+; GFX950-NEXT:    v_and_b32_e32 v12, 0xffff0000, v27
+; GFX950-NEXT:    v_fmac_f32_e32 v34, v13, v29
+; GFX950-NEXT:    v_fmac_f32_e32 v55, v57, v12
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v12, 16, v36
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v27
 ; GFX950-NEXT:    v_fmac_f32_e32 v12, v11, v13
 ; GFX950-NEXT:    scratch_load_dword v13, off, s32 offset:4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v31
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
-; GFX950-NEXT:    v_and_b32_e32 v58, 0xffff0000, v33
+; GFX950-NEXT:    v_and_b32_e32 v47, 0xffff0000, v33
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v33, 16, v33
 ; GFX950-NEXT:    v_fmac_f32_e32 v31, v15, v32
 ; GFX950-NEXT:    v_fmac_f32_e32 v33, v14, v30
@@ -48892,14 +48885,13 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX950-NEXT:    v_fmac_f32_e32 v19, v2, v18
 ; GFX950-NEXT:    s_waitcnt vmcnt(1)
-; GFX950-NEXT:    v_and_b32_e32 v2, 0xffff0000, v54
+; GFX950-NEXT:    v_and_b32_e32 v2, 0xffff0000, v58
 ; GFX950-NEXT:    v_and_b32_e32 v18, 0xffff0000, v17
 ; GFX950-NEXT:    v_and_b32_e32 v26, 0xffff0000, v1
 ; GFX950-NEXT:    v_fmac_f32_e32 v2, v26, v18
-; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v54
+; GFX950-NEXT:    v_lshlrev_b32_e32 v18, 16, v58
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_and_b32_e32 v57, 0xffff0000, v36
 ; GFX950-NEXT:    v_fmac_f32_e32 v18, v1, v17
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
 ; GFX950-NEXT:    v_and_b32_e32 v1, 0xffff0000, v13
@@ -48908,10 +48900,9 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_fmac_f32_e32 v58, v41, v40
-; GFX950-NEXT:    v_fmac_f32_e32 v59, v43, v42
-; GFX950-NEXT:    v_fmac_f32_e32 v55, v45, v44
-; GFX950-NEXT:    v_fmac_f32_e32 v57, v47, v46
+; GFX950-NEXT:    v_fmac_f32_e32 v47, v41, v40
+; GFX950-NEXT:    v_fmac_f32_e32 v56, v43, v42
+; GFX950-NEXT:    v_fmac_f32_e32 v54, v45, v44
 ; GFX950-NEXT:    v_fmac_f32_e32 v1, v26, v17
 ; GFX950-NEXT:    v_fmac_f32_e32 v13, v0, v16
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v13, v1
@@ -48925,12 +48916,11 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v8, v25, v9
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v9, v15, v10
 ; GFX950-NEXT:    v_cvt_pk_bf16_f32 v10, v14, v11
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v12, v57
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v35, v55
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v34, v59
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v33, v58
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v31, v56
-; GFX950-NEXT:    v_accvgpr_read_b32 v59, a11 ; Reload Reuse
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v11, v12, v55
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v12, v35, v54
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v13, v34, v56
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v14, v33, v47
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v15, v31, v46
 ; GFX950-NEXT:    v_accvgpr_read_b32 v58, a10 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_read_b32 v57, a9 ; Reload Reuse
 ; GFX950-NEXT:    v_accvgpr_read_b32 v56, a8 ; Reload Reuse
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
index 918fa1364d853..4bda91924871c 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
@@ -2217,99 +2217,86 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
 ; GCN-LABEL: v_copysign_v32bf16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GCN-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GCN-NEXT:    s_waitcnt expcnt(1)
-; GCN-NEXT:    v_and_b32_e32 v58, 0x8000, v17
-; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    v_and_b32_e32 v33, 0x7fff, v14
+; GCN-NEXT:    s_waitcnt expcnt(2)
+; GCN-NEXT:    v_and_b32_e32 v40, 0x8000, v30
+; GCN-NEXT:    v_or_b32_e32 v37, v33, v40
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v29
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v13
+; GCN-NEXT:    v_or_b32_e32 v39, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v28
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v12
+; GCN-NEXT:    v_or_b32_e32 v50, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v27
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v11
+; GCN-NEXT:    v_or_b32_e32 v51, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v26
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v10
+; GCN-NEXT:    v_or_b32_e32 v54, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v25
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v9
+; GCN-NEXT:    v_or_b32_e32 v55, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v24
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v8
 ; GCN-NEXT:    s_waitcnt expcnt(0)
-; GCN-NEXT:    v_and_b32_e32 v59, 0x7fff, v1
-; GCN-NEXT:    v_and_b32_e32 v17, 0x8000, v17
-; GCN-NEXT:    v_bfe_u32 v1, v1, 16, 15
-; GCN-NEXT:    v_and_b32_e32 v38, 0x8000, v27
-; GCN-NEXT:    v_and_b32_e32 v39, 0x7fff, v11
-; GCN-NEXT:    v_and_b32_e32 v48, 0x8000, v26
-; GCN-NEXT:    v_and_b32_e32 v49, 0x7fff, v10
-; GCN-NEXT:    v_and_b32_e32 v50, 0x8000, v25
-; GCN-NEXT:    v_and_b32_e32 v51, 0x7fff, v9
-; GCN-NEXT:    v_and_b32_e32 v40, 0x8000, v22
-; GCN-NEXT:    v_and_b32_e32 v41, 0x7fff, v6
-; GCN-NEXT:    v_or_b32_e32 v1, v1, v17
-; GCN-NEXT:    v_and_b32_e32 v52, 0x8000, v24
-; GCN-NEXT:    v_and_b32_e32 v53, 0x7fff, v8
-; GCN-NEXT:    v_and_b32_e32 v54, 0x8000, v23
-; GCN-NEXT:    v_and_b32_e32 v55, 0x7fff, v7
-; GCN-NEXT:    v_and_b32_e32 v42, 0x8000, v21
-; GCN-NEXT:    v_and_b32_e32 v43, 0x7fff, v5
-; GCN-NEXT:    v_and_b32_e32 v44, 0x8000, v20
-; GCN-NEXT:    v_and_b32_e32 v45, 0x7fff, v4
-; GCN-NEXT:    v_and_b32_e32 v46, 0x8000, v19
-; GCN-NEXT:    v_and_b32_e32 v47, 0x7fff, v3
-; GCN-NEXT:    v_and_b32_e32 v56, 0x8000, v18
-; GCN-NEXT:    v_and_b32_e32 v57, 0x7fff, v2
-; GCN-NEXT:    v_or_b32_e32 v38, v39, v38
-; GCN-NEXT:    v_or_b32_e32 v39, v49, v48
-; GCN-NEXT:    v_or_b32_e32 v48, v51, v50
-; GCN-NEXT:    v_or_b32_e32 v51, v41, v40
-; GCN-NEXT:    v_or_b32_e32 v40, v59, v58
-; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GCN-NEXT:    v_or_b32_e32 v49, v53, v52
-; GCN-NEXT:    v_or_b32_e32 v50, v55, v54
-; GCN-NEXT:    v_or_b32_e32 v52, v43, v42
-; GCN-NEXT:    v_or_b32_e32 v53, v45, v44
-; GCN-NEXT:    v_or_b32_e32 v54, v47, v46
-; GCN-NEXT:    v_or_b32_e32 v55, v57, v56
-; GCN-NEXT:    v_or_b32_e32 v1, v40, v1
-; GCN-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-NEXT:    v_or_b32_e32 v42, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v23
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v7
+; GCN-NEXT:    v_or_b32_e32 v32, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v22
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v6
+; GCN-NEXT:    v_or_b32_e32 v34, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v21
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v5
 ; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GCN-NEXT:    v_and_b32_e32 v32, 0x8000, v30
-; GCN-NEXT:    v_and_b32_e32 v33, 0x7fff, v14
-; GCN-NEXT:    v_and_b32_e32 v34, 0x8000, v29
-; GCN-NEXT:    v_and_b32_e32 v35, 0x7fff, v13
-; GCN-NEXT:    v_or_b32_e32 v32, v33, v32
-; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v16
-; GCN-NEXT:    v_or_b32_e32 v34, v35, v34
-; GCN-NEXT:    v_and_b32_e32 v35, 0x7fff, v0
-; GCN-NEXT:    v_and_b32_e32 v36, 0x8000, v28
-; GCN-NEXT:    v_and_b32_e32 v37, 0x7fff, v12
-; GCN-NEXT:    v_or_b32_e32 v33, v35, v33
+; GCN-NEXT:    v_or_b32_e32 v35, v40, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v20
+; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
+; GCN-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
+; GCN-NEXT:    v_and_b32_e32 v40, 0x7fff, v4
+; GCN-NEXT:    v_and_b32_e32 v24, 0x8000, v24
+; GCN-NEXT:    v_bfe_u32 v8, v8, 16, 15
+; GCN-NEXT:    v_and_b32_e32 v20, 0x8000, v20
+; GCN-NEXT:    v_bfe_u32 v4, v4, 16, 15
+; GCN-NEXT:    v_or_b32_e32 v8, v8, v24
+; GCN-NEXT:    v_or_b32_e32 v4, v4, v20
+; GCN-NEXT:    v_and_b32_e32 v36, 0x8000, v17
+; GCN-NEXT:    v_and_b32_e32 v38, 0x7fff, v1
+; GCN-NEXT:    v_and_b32_e32 v41, 0x8000, v16
+; GCN-NEXT:    v_and_b32_e32 v52, 0x7fff, v0
+; GCN-NEXT:    v_or_b32_e32 v40, v40, v33
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GCN-NEXT:    v_or_b32_e32 v36, v38, v36
+; GCN-NEXT:    v_or_b32_e32 v38, v52, v41
+; GCN-NEXT:    v_or_b32_e32 v4, v40, v4
+; GCN-NEXT:    v_or_b32_e32 v8, v42, v8
+; GCN-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-NEXT:    v_and_b32_e32 v49, 0x7fff, v3
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v19
+; GCN-NEXT:    v_and_b32_e32 v48, 0x7fff, v2
+; GCN-NEXT:    v_or_b32_e32 v49, v49, v33
+; GCN-NEXT:    v_and_b32_e32 v33, 0x8000, v18
+; GCN-NEXT:    v_or_b32_e32 v33, v48, v33
 ; GCN-NEXT:    v_lshrrev_b32_e32 v30, 16, v30
 ; GCN-NEXT:    v_lshrrev_b32_e32 v29, 16, v29
 ; GCN-NEXT:    v_lshrrev_b32_e32 v28, 16, v28
 ; GCN-NEXT:    v_lshrrev_b32_e32 v27, 16, v27
 ; GCN-NEXT:    v_lshrrev_b32_e32 v26, 16, v26
 ; GCN-NEXT:    v_lshrrev_b32_e32 v25, 16, v25
-; GCN-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
 ; GCN-NEXT:    v_lshrrev_b32_e32 v23, 16, v23
 ; GCN-NEXT:    v_lshrrev_b32_e32 v22, 16, v22
 ; GCN-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
-; GCN-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
 ; GCN-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
 ; GCN-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GCN-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
 ; GCN-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GCN-NEXT:    v_or_b32_e32 v36, v37, v36
-; GCN-NEXT:    v_and_b32_e32 v37, 0x7fff, v15
+; GCN-NEXT:    v_and_b32_e32 v53, 0x7fff, v15
 ; GCN-NEXT:    v_bfe_u32 v15, v15, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v30, 0x8000, v30
 ; GCN-NEXT:    v_bfe_u32 v14, v14, 16, 15
@@ -2323,20 +2310,18 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
 ; GCN-NEXT:    v_bfe_u32 v10, v10, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v25, 0x8000, v25
 ; GCN-NEXT:    v_bfe_u32 v9, v9, 16, 15
-; GCN-NEXT:    v_and_b32_e32 v24, 0x8000, v24
-; GCN-NEXT:    v_bfe_u32 v8, v8, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v23, 0x8000, v23
 ; GCN-NEXT:    v_bfe_u32 v7, v7, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v22, 0x8000, v22
 ; GCN-NEXT:    v_bfe_u32 v6, v6, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v21, 0x8000, v21
 ; GCN-NEXT:    v_bfe_u32 v5, v5, 16, 15
-; GCN-NEXT:    v_and_b32_e32 v20, 0x8000, v20
-; GCN-NEXT:    v_bfe_u32 v4, v4, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v19, 0x8000, v19
 ; GCN-NEXT:    v_bfe_u32 v3, v3, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v18, 0x8000, v18
 ; GCN-NEXT:    v_bfe_u32 v2, v2, 16, 15
+; GCN-NEXT:    v_and_b32_e32 v17, 0x8000, v17
+; GCN-NEXT:    v_bfe_u32 v1, v1, 16, 15
 ; GCN-NEXT:    v_and_b32_e32 v16, 0x8000, v16
 ; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 15
 ; GCN-NEXT:    v_or_b32_e32 v14, v14, v30
@@ -2345,146 +2330,131 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
 ; GCN-NEXT:    v_or_b32_e32 v11, v11, v27
 ; GCN-NEXT:    v_or_b32_e32 v10, v10, v26
 ; GCN-NEXT:    v_or_b32_e32 v9, v9, v25
-; GCN-NEXT:    v_or_b32_e32 v8, v8, v24
 ; GCN-NEXT:    v_or_b32_e32 v7, v7, v23
 ; GCN-NEXT:    v_or_b32_e32 v6, v6, v22
 ; GCN-NEXT:    v_or_b32_e32 v5, v5, v21
-; GCN-NEXT:    v_or_b32_e32 v4, v4, v20
+; GCN-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NEXT:    v_and_b32_e32 v48, 0x8000, v31
+; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
+; GCN-NEXT:    v_and_b32_e32 v31, 0x8000, v31
+; GCN-NEXT:    v_or_b32_e32 v15, v15, v31
 ; GCN-NEXT:    v_or_b32_e32 v3, v3, v19
 ; GCN-NEXT:    v_or_b32_e32 v2, v2, v18
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v17
 ; GCN-NEXT:    v_or_b32_e32 v0, v0, v16
+; GCN-NEXT:    v_or_b32_e32 v48, v53, v48
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GCN-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GCN-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GCN-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GCN-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_and_b32_e32 v35, 0x8000, v31
-; GCN-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
-; GCN-NEXT:    v_and_b32_e32 v31, 0x8000, v31
-; GCN-NEXT:    v_or_b32_e32 v15, v15, v31
-; GCN-NEXT:    v_or_b32_e32 v35, v37, v35
 ; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; GCN-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GCN-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GCN-NEXT:    v_or_b32_e32 v0, v33, v0
-; GCN-NEXT:    v_or_b32_e32 v2, v55, v2
-; GCN-NEXT:    v_or_b32_e32 v3, v54, v3
-; GCN-NEXT:    v_or_b32_e32 v4, v53, v4
-; GCN-NEXT:    v_or_b32_e32 v5, v52, v5
-; GCN-NEXT:    v_or_b32_e32 v6, v51, v6
-; GCN-NEXT:    v_or_b32_e32 v7, v50, v7
-; GCN-NEXT:    v_or_b32_e32 v8, v49, v8
-; GCN-NEXT:    v_or_b32_e32 v9, v48, v9
-; GCN-NEXT:    v_or_b32_e32 v10, v39, v10
-; GCN-NEXT:    v_or_b32_e32 v11, v38, v11
-; GCN-NEXT:    v_or_b32_e32 v12, v36, v12
-; GCN-NEXT:    v_or_b32_e32 v13, v34, v13
-; GCN-NEXT:    v_or_b32_e32 v14, v32, v14
-; GCN-NEXT:    v_or_b32_e32 v15, v35, v15
+; GCN-NEXT:    v_or_b32_e32 v0, v38, v0
+; GCN-NEXT:    v_or_b32_e32 v1, v36, v1
+; GCN-NEXT:    v_or_b32_e32 v2, v33, v2
+; GCN-NEXT:    v_or_b32_e32 v3, v49, v3
+; GCN-NEXT:    v_or_b32_e32 v5, v35, v5
+; GCN-NEXT:    v_or_b32_e32 v6, v34, v6
+; GCN-NEXT:    v_or_b32_e32 v7, v32, v7
+; GCN-NEXT:    v_or_b32_e32 v9, v55, v9
+; GCN-NEXT:    v_or_b32_e32 v10, v54, v10
+; GCN-NEXT:    v_or_b32_e32 v11, v51, v11
+; GCN-NEXT:    v_or_b32_e32 v12, v50, v12
+; GCN-NEXT:    v_or_b32_e32 v13, v39, v13
+; GCN-NEXT:    v_or_b32_e32 v14, v37, v14
+; GCN-NEXT:    v_or_b32_e32 v15, v48, v15
+; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: v_copysign_v32bf16:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX7-NEXT:    v_and_b32_e32 v58, 0x8000, v17
-; GFX7-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_and_b32_e32 v59, 0x7fff, v1
-; GFX7-NEXT:    v_and_b32_e32 v17, 0x8000, v17
-; GFX7-NEXT:    v_bfe_u32 v1, v1, 16, 15
-; GFX7-NEXT:    v_and_b32_e32 v38, 0x8000, v27
-; GFX7-NEXT:    v_and_b32_e32 v39, 0x7fff, v11
-; GFX7-NEXT:    v_and_b32_e32 v48, 0x8000, v26
-; GFX7-NEXT:    v_and_b32_e32 v49, 0x7fff, v10
-; GFX7-NEXT:    v_and_b32_e32 v50, 0x8000, v25
-; GFX7-NEXT:    v_and_b32_e32 v51, 0x7fff, v9
-; GFX7-NEXT:    v_and_b32_e32 v40, 0x8000, v22
-; GFX7-NEXT:    v_and_b32_e32 v41, 0x7fff, v6
-; GFX7-NEXT:    v_or_b32_e32 v1, v1, v17
-; GFX7-NEXT:    v_and_b32_e32 v52, 0x8000, v24
-; GFX7-NEXT:    v_and_b32_e32 v53, 0x7fff, v8
-; GFX7-NEXT:    v_and_b32_e32 v54, 0x8000, v23
-; GFX7-NEXT:    v_and_b32_e32 v55, 0x7fff, v7
-; GFX7-NEXT:    v_and_b32_e32 v42, 0x8000, v21
-; GFX7-NEXT:    v_and_b32_e32 v43, 0x7fff, v5
-; GFX7-NEXT:    v_and_b32_e32 v44, 0x8000, v20
-; GFX7-NEXT:    v_and_b32_e32 v45, 0x7fff, v4
-; GFX7-NEXT:    v_and_b32_e32 v46, 0x8000, v19
-; GFX7-NEXT:    v_and_b32_e32 v47, 0x7fff, v3
-; GFX7-NEXT:    v_and_b32_e32 v56, 0x8000, v18
-; GFX7-NEXT:    v_and_b32_e32 v57, 0x7fff, v2
-; GFX7-NEXT:    v_or_b32_e32 v38, v39, v38
-; GFX7-NEXT:    v_or_b32_e32 v39, v49, v48
-; GFX7-NEXT:    v_or_b32_e32 v48, v51, v50
-; GFX7-NEXT:    v_or_b32_e32 v51, v41, v40
-; GFX7-NEXT:    v_or_b32_e32 v40, v59, v58
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_or_b32_e32 v49, v53, v52
-; GFX7-NEXT:    v_or_b32_e32 v50, v55, v54
-; GFX7-NEXT:    v_or_b32_e32 v52, v43, v42
-; GFX7-NEXT:    v_or_b32_e32 v53, v45, v44
-; GFX7-NEXT:    v_or_b32_e32 v54, v47, v46
-; GFX7-NEXT:    v_or_b32_e32 v55, v57, v56
-; GFX7-NEXT:    v_or_b32_e32 v1, v40, v1
-; GFX7-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX7-NEXT:    v_and_b32_e32 v32, 0x8000, v30
+; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX7-NEXT:    v_and_b32_e32 v33, 0x7fff, v14
-; GFX7-NEXT:    v_and_b32_e32 v34, 0x8000, v29
-; GFX7-NEXT:    v_and_b32_e32 v35, 0x7fff, v13
-; GFX7-NEXT:    v_or_b32_e32 v32, v33, v32
-; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v16
-; GFX7-NEXT:    v_or_b32_e32 v34, v35, v34
-; GFX7-NEXT:    v_and_b32_e32 v35, 0x7fff, v0
-; GFX7-NEXT:    v_and_b32_e32 v36, 0x8000, v28
-; GFX7-NEXT:    v_and_b32_e32 v37, 0x7fff, v12
-; GFX7-NEXT:    v_or_b32_e32 v33, v35, v33
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x8000, v30
+; GFX7-NEXT:    v_or_b32_e32 v37, v33, v40
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v29
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v13
+; GFX7-NEXT:    v_or_b32_e32 v39, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v28
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v12
+; GFX7-NEXT:    v_or_b32_e32 v50, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v27
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v11
+; GFX7-NEXT:    v_or_b32_e32 v51, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v26
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v10
+; GFX7-NEXT:    v_or_b32_e32 v54, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v25
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v9
+; GFX7-NEXT:    v_or_b32_e32 v55, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v24
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v8
+; GFX7-NEXT:    v_or_b32_e32 v42, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v23
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v7
+; GFX7-NEXT:    v_or_b32_e32 v32, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v22
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v6
+; GFX7-NEXT:    v_or_b32_e32 v34, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v21
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v5
+; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GFX7-NEXT:    v_or_b32_e32 v35, v40, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v20
+; GFX7-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
+; GFX7-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
+; GFX7-NEXT:    v_and_b32_e32 v40, 0x7fff, v4
+; GFX7-NEXT:    v_and_b32_e32 v24, 0x8000, v24
+; GFX7-NEXT:    v_bfe_u32 v8, v8, 16, 15
+; GFX7-NEXT:    v_and_b32_e32 v20, 0x8000, v20
+; GFX7-NEXT:    v_bfe_u32 v4, v4, 16, 15
+; GFX7-NEXT:    v_or_b32_e32 v8, v8, v24
+; GFX7-NEXT:    v_or_b32_e32 v4, v4, v20
+; GFX7-NEXT:    v_and_b32_e32 v36, 0x8000, v17
+; GFX7-NEXT:    v_and_b32_e32 v38, 0x7fff, v1
+; GFX7-NEXT:    v_and_b32_e32 v41, 0x8000, v16
+; GFX7-NEXT:    v_and_b32_e32 v52, 0x7fff, v0
+; GFX7-NEXT:    v_or_b32_e32 v40, v40, v33
+; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX7-NEXT:    v_or_b32_e32 v36, v38, v36
+; GFX7-NEXT:    v_or_b32_e32 v38, v52, v41
+; GFX7-NEXT:    v_or_b32_e32 v4, v40, v4
+; GFX7-NEXT:    v_or_b32_e32 v8, v42, v8
+; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT:    v_and_b32_e32 v49, 0x7fff, v3
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v19
+; GFX7-NEXT:    v_and_b32_e32 v48, 0x7fff, v2
+; GFX7-NEXT:    v_or_b32_e32 v49, v49, v33
+; GFX7-NEXT:    v_and_b32_e32 v33, 0x8000, v18
+; GFX7-NEXT:    v_or_b32_e32 v33, v48, v33
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v30, 16, v30
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v29, 16, v29
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v28, 16, v28
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v27, 16, v27
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v26, 16, v26
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v25, 16, v25
-; GFX7-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v23, 16, v23
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v22, 16, v22
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
-; GFX7-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GFX7-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GFX7-NEXT:    v_or_b32_e32 v36, v37, v36
-; GFX7-NEXT:    v_and_b32_e32 v37, 0x7fff, v15
+; GFX7-NEXT:    v_and_b32_e32 v53, 0x7fff, v15
 ; GFX7-NEXT:    v_bfe_u32 v15, v15, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v30, 0x8000, v30
 ; GFX7-NEXT:    v_bfe_u32 v14, v14, 16, 15
@@ -2498,20 +2468,18 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
 ; GFX7-NEXT:    v_bfe_u32 v10, v10, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v25, 0x8000, v25
 ; GFX7-NEXT:    v_bfe_u32 v9, v9, 16, 15
-; GFX7-NEXT:    v_and_b32_e32 v24, 0x8000, v24
-; GFX7-NEXT:    v_bfe_u32 v8, v8, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v23, 0x8000, v23
 ; GFX7-NEXT:    v_bfe_u32 v7, v7, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v22, 0x8000, v22
 ; GFX7-NEXT:    v_bfe_u32 v6, v6, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v21, 0x8000, v21
 ; GFX7-NEXT:    v_bfe_u32 v5, v5, 16, 15
-; GFX7-NEXT:    v_and_b32_e32 v20, 0x8000, v20
-; GFX7-NEXT:    v_bfe_u32 v4, v4, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v19, 0x8000, v19
 ; GFX7-NEXT:    v_bfe_u32 v3, v3, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v18, 0x8000, v18
 ; GFX7-NEXT:    v_bfe_u32 v2, v2, 16, 15
+; GFX7-NEXT:    v_and_b32_e32 v17, 0x8000, v17
+; GFX7-NEXT:    v_bfe_u32 v1, v1, 16, 15
 ; GFX7-NEXT:    v_and_b32_e32 v16, 0x8000, v16
 ; GFX7-NEXT:    v_bfe_u32 v0, v0, 16, 15
 ; GFX7-NEXT:    v_or_b32_e32 v14, v14, v30
@@ -2520,50 +2488,48 @@ define <32 x bfloat> @v_copysign_v32bf16(<32 x bfloat> %mag, <32 x bfloat> %sign
 ; GFX7-NEXT:    v_or_b32_e32 v11, v11, v27
 ; GFX7-NEXT:    v_or_b32_e32 v10, v10, v26
 ; GFX7-NEXT:    v_or_b32_e32 v9, v9, v25
-; GFX7-NEXT:    v_or_b32_e32 v8, v8, v24
 ; GFX7-NEXT:    v_or_b32_e32 v7, v7, v23
 ; GFX7-NEXT:    v_or_b32_e32 v6, v6, v22
 ; GFX7-NEXT:    v_or_b32_e32 v5, v5, v21
-; GFX7-NEXT:    v_or_b32_e32 v4, v4, v20
+; GFX7-NEXT:    s_waitcnt vmcnt(3)
+; GFX7-NEXT:    v_and_b32_e32 v48, 0x8000, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
+; GFX7-NEXT:    v_and_b32_e32 v31, 0x8000, v31
+; GFX7-NEXT:    v_or_b32_e32 v15, v15, v31
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v19
 ; GFX7-NEXT:    v_or_b32_e32 v2, v2, v18
+; GFX7-NEXT:    v_or_b32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_or_b32_e32 v0, v0, v16
+; GFX7-NEXT:    v_or_b32_e32 v48, v53, v48
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_and_b32_e32 v35, 0x8000, v31
-; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
-; GFX7-NEXT:    v_and_b32_e32 v31, 0x8000, v31
-; GFX7-NEXT:    v_or_b32_e32 v15, v15, v31
-; GFX7-NEXT:    v_or_b32_e32 v35, v37, v35
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX7-NEXT:    v_or_b32_e32 v0, v33, v0
-; GFX7-NEXT:    v_or_b32_e32 v2, v55, v2
-; GFX7-NEXT:    v_or_b32_e32 v3, v54, v3
-; GFX7-NEXT:    v_or_b32_e32 v4, v53, v4
-; GFX7-NEXT:    v_or_b32_e32 v5, v52, v5
-; GFX7-NEXT:    v_or_b32_e32 v6, v51, v6
-; GFX7-NEXT:    v_or_b32_e32 v7, v50, v7
-; GFX7-NEXT:    v_or_b32_e32 v8, v49, v8
-; GFX7-NEXT:    v_or_b32_e32 v9, v48, v9
-; GFX7-NEXT:    v_or_b32_e32 v10, v39, v10
-; GFX7-NEXT:    v_or_b32_e32 v11, v38, v11
-; GFX7-NEXT:    v_or_b32_e32 v12, v36, v12
-; GFX7-NEXT:    v_or_b32_e32 v13, v34, v13
-; GFX7-NEXT:    v_or_b32_e32 v14, v32, v14
-; GFX7-NEXT:    v_or_b32_e32 v15, v35, v15
+; GFX7-NEXT:    v_or_b32_e32 v0, v38, v0
+; GFX7-NEXT:    v_or_b32_e32 v1, v36, v1
+; GFX7-NEXT:    v_or_b32_e32 v2, v33, v2
+; GFX7-NEXT:    v_or_b32_e32 v3, v49, v3
+; GFX7-NEXT:    v_or_b32_e32 v5, v35, v5
+; GFX7-NEXT:    v_or_b32_e32 v6, v34, v6
+; GFX7-NEXT:    v_or_b32_e32 v7, v32, v7
+; GFX7-NEXT:    v_or_b32_e32 v9, v55, v9
+; GFX7-NEXT:    v_or_b32_e32 v10, v54, v10
+; GFX7-NEXT:    v_or_b32_e32 v11, v51, v11
+; GFX7-NEXT:    v_or_b32_e32 v12, v50, v12
+; GFX7-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX7-NEXT:    v_or_b32_e32 v14, v37, v14
+; GFX7-NEXT:    v_or_b32_e32 v15, v48, v15
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_copysign_v32bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index ba404a8d573ab..13bb2393e1449 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -2101,99 +2101,86 @@ define <32 x half> @v_copysign_v32f32(<32 x half> %mag, <32 x half> %sign) {
 ; SI-LABEL: v_copysign_v32f32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; SI-NEXT:    s_waitcnt expcnt(1)
-; SI-NEXT:    v_and_b32_e32 v58, 0x8000, v17
-; SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; SI-NEXT:    v_and_b32_e32 v33, 0x7fff, v14
+; SI-NEXT:    s_waitcnt expcnt(2)
+; SI-NEXT:    v_and_b32_e32 v40, 0x8000, v30
+; SI-NEXT:    v_or_b32_e32 v37, v33, v40
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v29
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v13
+; SI-NEXT:    v_or_b32_e32 v39, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v28
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v12
+; SI-NEXT:    v_or_b32_e32 v50, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v27
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v11
+; SI-NEXT:    v_or_b32_e32 v51, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v26
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v10
+; SI-NEXT:    v_or_b32_e32 v54, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v25
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v9
+; SI-NEXT:    v_or_b32_e32 v55, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v24
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v8
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v59, 0x7fff, v1
-; SI-NEXT:    v_and_b32_e32 v17, 0x8000, v17
-; SI-NEXT:    v_bfe_u32 v1, v1, 16, 15
-; SI-NEXT:    v_and_b32_e32 v38, 0x8000, v27
-; SI-NEXT:    v_and_b32_e32 v39, 0x7fff, v11
-; SI-NEXT:    v_and_b32_e32 v48, 0x8000, v26
-; SI-NEXT:    v_and_b32_e32 v49, 0x7fff, v10
-; SI-NEXT:    v_and_b32_e32 v50, 0x8000, v25
-; SI-NEXT:    v_and_b32_e32 v51, 0x7fff, v9
-; SI-NEXT:    v_and_b32_e32 v40, 0x8000, v22
-; SI-NEXT:    v_and_b32_e32 v41, 0x7fff, v6
-; SI-NEXT:    v_or_b32_e32 v1, v1, v17
-; SI-NEXT:    v_and_b32_e32 v52, 0x8000, v24
-; SI-NEXT:    v_and_b32_e32 v53, 0x7fff, v8
-; SI-NEXT:    v_and_b32_e32 v54, 0x8000, v23
-; SI-NEXT:    v_and_b32_e32 v55, 0x7fff, v7
-; SI-NEXT:    v_and_b32_e32 v42, 0x8000, v21
-; SI-NEXT:    v_and_b32_e32 v43, 0x7fff, v5
-; SI-NEXT:    v_and_b32_e32 v44, 0x8000, v20
-; SI-NEXT:    v_and_b32_e32 v45, 0x7fff, v4
-; SI-NEXT:    v_and_b32_e32 v46, 0x8000, v19
-; SI-NEXT:    v_and_b32_e32 v47, 0x7fff, v3
-; SI-NEXT:    v_and_b32_e32 v56, 0x8000, v18
-; SI-NEXT:    v_and_b32_e32 v57, 0x7fff, v2
-; SI-NEXT:    v_or_b32_e32 v38, v39, v38
-; SI-NEXT:    v_or_b32_e32 v39, v49, v48
-; SI-NEXT:    v_or_b32_e32 v48, v51, v50
-; SI-NEXT:    v_or_b32_e32 v51, v41, v40
-; SI-NEXT:    v_or_b32_e32 v40, v59, v58
-; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT:    v_or_b32_e32 v49, v53, v52
-; SI-NEXT:    v_or_b32_e32 v50, v55, v54
-; SI-NEXT:    v_or_b32_e32 v52, v43, v42
-; SI-NEXT:    v_or_b32_e32 v53, v45, v44
-; SI-NEXT:    v_or_b32_e32 v54, v47, v46
-; SI-NEXT:    v_or_b32_e32 v55, v57, v56
-; SI-NEXT:    v_or_b32_e32 v1, v40, v1
-; SI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT:    v_or_b32_e32 v42, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v23
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v7
+; SI-NEXT:    v_or_b32_e32 v32, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v22
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v6
+; SI-NEXT:    v_or_b32_e32 v34, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v21
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v5
 ; SI-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; SI-NEXT:    v_and_b32_e32 v32, 0x8000, v30
-; SI-NEXT:    v_and_b32_e32 v33, 0x7fff, v14
-; SI-NEXT:    v_and_b32_e32 v34, 0x8000, v29
-; SI-NEXT:    v_and_b32_e32 v35, 0x7fff, v13
-; SI-NEXT:    v_or_b32_e32 v32, v33, v32
-; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v16
-; SI-NEXT:    v_or_b32_e32 v34, v35, v34
-; SI-NEXT:    v_and_b32_e32 v35, 0x7fff, v0
-; SI-NEXT:    v_and_b32_e32 v36, 0x8000, v28
-; SI-NEXT:    v_and_b32_e32 v37, 0x7fff, v12
-; SI-NEXT:    v_or_b32_e32 v33, v35, v33
+; SI-NEXT:    v_or_b32_e32 v35, v40, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v20
+; SI-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
+; SI-NEXT:    v_and_b32_e32 v40, 0x7fff, v4
+; SI-NEXT:    v_and_b32_e32 v24, 0x8000, v24
+; SI-NEXT:    v_bfe_u32 v8, v8, 16, 15
+; SI-NEXT:    v_and_b32_e32 v20, 0x8000, v20
+; SI-NEXT:    v_bfe_u32 v4, v4, 16, 15
+; SI-NEXT:    v_or_b32_e32 v8, v8, v24
+; SI-NEXT:    v_or_b32_e32 v4, v4, v20
+; SI-NEXT:    v_and_b32_e32 v36, 0x8000, v17
+; SI-NEXT:    v_and_b32_e32 v38, 0x7fff, v1
+; SI-NEXT:    v_and_b32_e32 v41, 0x8000, v16
+; SI-NEXT:    v_and_b32_e32 v52, 0x7fff, v0
+; SI-NEXT:    v_or_b32_e32 v40, v40, v33
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT:    v_or_b32_e32 v36, v38, v36
+; SI-NEXT:    v_or_b32_e32 v38, v52, v41
+; SI-NEXT:    v_or_b32_e32 v4, v40, v4
+; SI-NEXT:    v_or_b32_e32 v8, v42, v8
+; SI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT:    v_and_b32_e32 v49, 0x7fff, v3
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v19
+; SI-NEXT:    v_and_b32_e32 v48, 0x7fff, v2
+; SI-NEXT:    v_or_b32_e32 v49, v49, v33
+; SI-NEXT:    v_and_b32_e32 v33, 0x8000, v18
+; SI-NEXT:    v_or_b32_e32 v33, v48, v33
 ; SI-NEXT:    v_lshrrev_b32_e32 v30, 16, v30
 ; SI-NEXT:    v_lshrrev_b32_e32 v29, 16, v29
 ; SI-NEXT:    v_lshrrev_b32_e32 v28, 16, v28
 ; SI-NEXT:    v_lshrrev_b32_e32 v27, 16, v27
 ; SI-NEXT:    v_lshrrev_b32_e32 v26, 16, v26
 ; SI-NEXT:    v_lshrrev_b32_e32 v25, 16, v25
-; SI-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
 ; SI-NEXT:    v_lshrrev_b32_e32 v23, 16, v23
 ; SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v22
 ; SI-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
-; SI-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
 ; SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
 ; SI-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
 ; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; SI-NEXT:    v_or_b32_e32 v36, v37, v36
-; SI-NEXT:    v_and_b32_e32 v37, 0x7fff, v15
+; SI-NEXT:    v_and_b32_e32 v53, 0x7fff, v15
 ; SI-NEXT:    v_bfe_u32 v15, v15, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v30, 0x8000, v30
 ; SI-NEXT:    v_bfe_u32 v14, v14, 16, 15
@@ -2207,20 +2194,18 @@ define <32 x half> @v_copysign_v32f32(<32 x half> %mag, <32 x half> %sign) {
 ; SI-NEXT:    v_bfe_u32 v10, v10, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v25, 0x8000, v25
 ; SI-NEXT:    v_bfe_u32 v9, v9, 16, 15
-; SI-NEXT:    v_and_b32_e32 v24, 0x8000, v24
-; SI-NEXT:    v_bfe_u32 v8, v8, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v23, 0x8000, v23
 ; SI-NEXT:    v_bfe_u32 v7, v7, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v22, 0x8000, v22
 ; SI-NEXT:    v_bfe_u32 v6, v6, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v21, 0x8000, v21
 ; SI-NEXT:    v_bfe_u32 v5, v5, 16, 15
-; SI-NEXT:    v_and_b32_e32 v20, 0x8000, v20
-; SI-NEXT:    v_bfe_u32 v4, v4, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v19, 0x8000, v19
 ; SI-NEXT:    v_bfe_u32 v3, v3, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v18, 0x8000, v18
 ; SI-NEXT:    v_bfe_u32 v2, v2, 16, 15
+; SI-NEXT:    v_and_b32_e32 v17, 0x8000, v17
+; SI-NEXT:    v_bfe_u32 v1, v1, 16, 15
 ; SI-NEXT:    v_and_b32_e32 v16, 0x8000, v16
 ; SI-NEXT:    v_bfe_u32 v0, v0, 16, 15
 ; SI-NEXT:    v_or_b32_e32 v14, v14, v30
@@ -2229,50 +2214,48 @@ define <32 x half> @v_copysign_v32f32(<32 x half> %mag, <32 x half> %sign) {
 ; SI-NEXT:    v_or_b32_e32 v11, v11, v27
 ; SI-NEXT:    v_or_b32_e32 v10, v10, v26
 ; SI-NEXT:    v_or_b32_e32 v9, v9, v25
-; SI-NEXT:    v_or_b32_e32 v8, v8, v24
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v23
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v22
 ; SI-NEXT:    v_or_b32_e32 v5, v5, v21
-; SI-NEXT:    v_or_b32_e32 v4, v4, v20
+; SI-NEXT:    s_waitcnt vmcnt(3)
+; SI-NEXT:    v_and_b32_e32 v48, 0x8000, v31
+; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT:    v_and_b32_e32 v31, 0x8000, v31
+; SI-NEXT:    v_or_b32_e32 v15, v15, v31
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v19
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v18
+; SI-NEXT:    v_or_b32_e32 v1, v1, v17
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v16
+; SI-NEXT:    v_or_b32_e32 v48, v53, v48
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; SI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; SI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v35, 0x8000, v31
-; SI-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT:    v_and_b32_e32 v31, 0x8000, v31
-; SI-NEXT:    v_or_b32_e32 v15, v15, v31
-; SI-NEXT:    v_or_b32_e32 v35, v37, v35
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; SI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
 ; SI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
 ; SI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
 ; SI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT:    v_or_b32_e32 v0, v33, v0
-; SI-NEXT:    v_or_b32_e32 v2, v55, v2
-; SI-NEXT:    v_or_b32_e32 v3, v54, v3
-; SI-NEXT:    v_or_b32_e32 v4, v53, v4
-; SI-NEXT:    v_or_b32_e32 v5, v52, v5
-; SI-NEXT:    v_or_b32_e32 v6, v51, v6
-; SI-NEXT:    v_or_b32_e32 v7, v50, v7
-; SI-NEXT:    v_or_b32_e32 v8, v49, v8
-; SI-NEXT:    v_or_b32_e32 v9, v48, v9
-; SI-NEXT:    v_or_b32_e32 v10, v39, v10
-; SI-NEXT:    v_or_b32_e32 v11, v38, v11
-; SI-NEXT:    v_or_b32_e32 v12, v36, v12
-; SI-NEXT:    v_or_b32_e32 v13, v34, v13
-; SI-NEXT:    v_or_b32_e32 v14, v32, v14
-; SI-NEXT:    v_or_b32_e32 v15, v35, v15
+; SI-NEXT:    v_or_b32_e32 v0, v38, v0
+; SI-NEXT:    v_or_b32_e32 v1, v36, v1
+; SI-NEXT:    v_or_b32_e32 v2, v33, v2
+; SI-NEXT:    v_or_b32_e32 v3, v49, v3
+; SI-NEXT:    v_or_b32_e32 v5, v35, v5
+; SI-NEXT:    v_or_b32_e32 v6, v34, v6
+; SI-NEXT:    v_or_b32_e32 v7, v32, v7
+; SI-NEXT:    v_or_b32_e32 v9, v55, v9
+; SI-NEXT:    v_or_b32_e32 v10, v54, v10
+; SI-NEXT:    v_or_b32_e32 v11, v51, v11
+; SI-NEXT:    v_or_b32_e32 v12, v50, v12
+; SI-NEXT:    v_or_b32_e32 v13, v39, v13
+; SI-NEXT:    v_or_b32_e32 v14, v37, v14
+; SI-NEXT:    v_or_b32_e32 v15, v48, v15
+; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_copysign_v32f32:
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 6101d89f31186..961945b355f22 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -29,61 +29,57 @@ body:             |
   ; CHECK-NEXT:   dead undef [[DEF:%[0-9]+]].sub0:vreg_64 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead undef [[DEF2:%[0-9]+]].sub0:vreg_64 = IMPLICIT_DEF
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX16_IMM renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
-  ; CHECK-NEXT:   renamable $sgpr24 = IMPLICIT_DEF implicit-def $sgpr25
-  ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+  ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+  ; CHECK-NEXT:   renamable $sgpr40 = IMPLICIT_DEF implicit-def $sgpr41
   ; CHECK-NEXT:   $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.6, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.5
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
   ; CHECK-NEXT:   S_CBRANCH_EXECNZ %bb.4, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s256), addrspace 4)
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   SI_RETURN
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.4:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr12 = IMPLICIT_DEF
-  ; CHECK-NEXT:   SI_SPILL_S512_SAVE renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
-  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr44 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = IMPLICIT_DEF
+  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, renamable $sgpr44_sgpr45_sgpr46_sgpr47, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
-  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], undef renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], undef renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5:
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
   ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   S_BRANCH %bb.7
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.6:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, undef renamable $sgpr4_sgpr5, implicit-def $scc
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.8, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.7:
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   renamable $sgpr41 = COPY undef renamable $sgpr40, implicit-def $sgpr40
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.7, implicit undef $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.6
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index 7dafdf5d0a3d2..56265bdbcd359 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -29,58 +29,55 @@ body:             |
   ; CHECK-NEXT:   dead renamable $sgpr5 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead undef [[DEF3:%[0-9]+]].sub1:vreg_64 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead renamable $sgpr5 = IMPLICIT_DEF
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
-  ; CHECK-NEXT:   renamable $sgpr24 = IMPLICIT_DEF implicit-def $sgpr25
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+  ; CHECK-NEXT:   renamable $sgpr40 = IMPLICIT_DEF implicit-def $sgpr41
   ; CHECK-NEXT:   $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.5, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.4
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $exec = S_MOV_B64_term undef renamable $sgpr4_sgpr5
   ; CHECK-NEXT:   S_CBRANCH_EXECNZ %bb.3, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s256), addrspace 4)
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_1:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   SI_RETURN
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr12 = IMPLICIT_DEF
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
-  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX16_IMM undef renamable $sgpr4_sgpr5, 0, 0 :: (invariant load (s512), align 32, addrspace 4)
+  ; CHECK-NEXT:   renamable $sgpr44 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = IMPLICIT_DEF
+  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, renamable $sgpr44_sgpr45_sgpr46_sgpr47, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
-  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], undef renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF3]], undef renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.4:
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
   ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr24_sgpr25_sgpr26_sgpr27, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_4:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr40_sgpr41_sgpr42_sgpr43, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; CHECK-NEXT:   S_BRANCH %bb.6
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5:
-  ; CHECK-NEXT:   liveins: $sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FFFF
+  ; CHECK-NEXT:   liveins: $sgpr40_sgpr41_sgpr42_sgpr43:0x000000000000000F, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, undef renamable $sgpr4_sgpr5, implicit-def $scc
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.7, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.6:
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
-  ; CHECK-NEXT:   renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
+  ; CHECK-NEXT:   dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+  ; CHECK-NEXT:   renamable $sgpr41 = COPY undef renamable $sgpr40, implicit-def $sgpr40
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.6, implicit undef $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.5
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/issue48473.mir b/llvm/test/CodeGen/AMDGPU/issue48473.mir
index d655338dabae9..ab8f8059ae467 100644
--- a/llvm/test/CodeGen/AMDGPU/issue48473.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue48473.mir
@@ -43,7 +43,7 @@
 # %25 to $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
 
 # CHECK-LABEL: name: issue48473
-# CHECK: S_NOP 0, implicit killed renamable $sgpr20_sgpr21_sgpr22_sgpr23, implicit killed renamable $sgpr88_sgpr89_sgpr90_sgpr91, implicit killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, implicit killed renamable $sgpr40_sgpr41_sgpr42_sgpr43, implicit killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, implicit killed renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, implicit killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, implicit killed renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit killed renamable $sgpr16_sgpr17_sgpr18_sgpr19, implicit killed renamable $sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, implicit killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, implicit killed renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr44_sgpr45_sgpr46_sgpr47, implicit killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
+# CHECK: S_NOP 0, implicit killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, implicit killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47, implicit killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, implicit killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, implicit killed renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, implicit killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, implicit killed renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, implicit killed renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, implicit killed renamable $sgpr96_sgpr97_sgpr98_sgpr99, implicit killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
 
 ---
 name:            issue48473
diff --git a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
index afa9cc9af9ca2..b0797a01817de 100644
--- a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
+++ b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
@@ -31,9 +31,9 @@ body:             |
     ; CHECK-LABEL: name: temp_vgpr_to_agpr_should_not_undo_split_with_remat
     ; CHECK: liveins: $vgpr0, $sgpr4_sgpr5
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
+    ; CHECK-NEXT: renamable $vgpr39 = IMPLICIT_DEF
     ; CHECK-NEXT: dead renamable $vgpr1 = IMPLICIT_DEF
-    ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+    ; CHECK-NEXT: renamable $vgpr48 = IMPLICIT_DEF
     ; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF
     ; CHECK-NEXT: renamable $vgpr3 = IMPLICIT_DEF
     ; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF
@@ -57,9 +57,9 @@ body:             |
     ; CHECK-NEXT: renamable $vgpr22 = IMPLICIT_DEF
     ; CHECK-NEXT: KILL killed renamable $vgpr2, killed renamable $vgpr3, killed renamable $vgpr4, killed renamable $vgpr5, killed renamable $vgpr6, killed renamable $vgpr7, killed renamable $vgpr8, killed renamable $vgpr9, killed renamable $vgpr10, killed renamable $vgpr11, killed renamable $vgpr12, killed renamable $vgpr13, killed renamable $vgpr14, killed renamable $vgpr15, killed renamable $vgpr16
     ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38
-    ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54
-    ; CHECK-NEXT: KILL killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr17, killed renamable $vgpr18, killed renamable $vgpr19, killed renamable $vgpr20, killed renamable $vgpr21, killed renamable $vgpr22
-    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38, implicit killed renamable $vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54
+    ; CHECK-NEXT: S_NOP 0, implicit-def renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: KILL killed renamable $vgpr39, killed renamable $vgpr48, killed renamable $vgpr17, killed renamable $vgpr18, killed renamable $vgpr19, killed renamable $vgpr20, killed renamable $vgpr21, killed renamable $vgpr22
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; CHECK-NEXT: S_ENDPGM 0
     %1:vgpr_32 = IMPLICIT_DEF
     %2:vgpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index 504739de571a9..40dc296ded423 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -7965,7 +7965,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_cmp_eq_f32_e64 s[8:9], 0, v39
 ; GFX8-NEXT:    s_and_b64 vcc, s[6:7], vcc
 ; GFX8-NEXT:    v_and_b32_e32 v33, 0xffff0000, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v42, v32, v31, vcc
 ; GFX8-NEXT:    s_and_b64 vcc, s[8:9], s[4:5]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -7978,9 +7978,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_and_b32_e32 v52, 0xffff0000, v21
 ; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
 ; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX8-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX8-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v53
 ; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v53
@@ -8158,76 +8155,79 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
 ; GFX8-NEXT:    v_cndmask_b32_e32 v52, v55, v54, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v52, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v52
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v55, v40
+; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v18
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v52, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v52
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v31, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v54, v52, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v52
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v54, v52, vcc
-; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v41
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v54
-; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
-; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v52, v31, v52, vcc
+; GFX8-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v17
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v42
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v40
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v31
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v53, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v0
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v16
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v53, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v53, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v53
+; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v31
+; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v53
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v53, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v53, v15, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v53
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v31
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v53
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v15
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v53, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v14
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
 ; GFX8-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v53
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v14
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v53, v31
 ; GFX8-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v30
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v53
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v14
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8238,8 +8238,8 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX8-NEXT:    v_lshlrev_b32_e32 v53, 16, v13
-; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v53, v30
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v13
+; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v31, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8346,9 +8346,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX8-NEXT:    v_cmp_gt_f32_e32 vcc, v24, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc
-; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v6
@@ -8472,7 +8469,10 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_or_b32_sdwa v12, v12, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v32
 ; GFX8-NEXT:    v_or_b32_sdwa v13, v13, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v42
+; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX8-NEXT:    v_or_b32_sdwa v14, v14, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v33
 ; GFX8-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -8523,7 +8523,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_cmp_eq_f32_e64 s[8:9], 0, v39
 ; GFX900-NEXT:    s_and_b64 vcc, s[6:7], vcc
 ; GFX900-NEXT:    v_and_b32_e32 v33, 0xffff0000, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v42, v32, v31, vcc
 ; GFX900-NEXT:    s_and_b64 vcc, s[8:9], s[4:5]
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -8536,9 +8536,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
 ; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX900-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX900-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v35, 16, v52
 ; GFX900-NEXT:    v_and_b32_e32 v36, 0xffff0000, v52
@@ -8716,76 +8713,79 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v53
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v55, v40
+; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v18
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v53
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v31, v55
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v53
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v41
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v54
-; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
-; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX900-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
+; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v17
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
-; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v42
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v40
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v55
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v55
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v31
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v0
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v16
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
+; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v31
+; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v52
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
+; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v52
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v15
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
 ; GFX900-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
-; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v52
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
+; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v31
 ; GFX900-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v14
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8796,8 +8796,8 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
-; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v13
+; GFX900-NEXT:    v_cmp_gt_f32_e32 vcc, v31, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8882,9 +8882,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc
@@ -9001,8 +8998,12 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], 0, v0
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX900-NEXT:    v_perm_b32 v14, v42, v14, s4
+; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
 ; GFX900-NEXT:    v_perm_b32 v0, v55, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v54, v1, s4
 ; GFX900-NEXT:    v_perm_b32 v2, v53, v2, s4
@@ -9017,7 +9018,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_perm_b32 v11, v35, v11, s4
 ; GFX900-NEXT:    v_perm_b32 v12, v34, v12, s4
 ; GFX900-NEXT:    v_perm_b32 v13, v32, v13, s4
-; GFX900-NEXT:    v_perm_b32 v14, v31, v14, s4
 ; GFX900-NEXT:    v_perm_b32 v15, v33, v15, s4
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -9067,7 +9067,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_cmp_eq_f32_e64 s[2:3], 0, v38
 ; GFX950-NEXT:    v_cmp_eq_f32_e64 s[4:5], 0, v39
 ; GFX950-NEXT:    s_and_b64 vcc, s[2:3], vcc
-; GFX950-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v42, v32, v31, vcc
 ; GFX950-NEXT:    s_and_b64 vcc, s[4:5], s[0:1]
 ; GFX950-NEXT:    v_cndmask_b32_e32 v32, v36, v35, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v33, v33
@@ -9079,9 +9079,6 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
 ; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v52
 ; GFX950-NEXT:    v_and_b32_e32 v36, 0xffff0000, v52
@@ -9280,91 +9277,93 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v53
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v18
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v53
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v53
 ; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v55, v40
-; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
+; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v31, v55
+; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
+; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
+; GFX950-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX950-NEXT:    v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    s_nop 1
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v17
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v31
+; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v31
+; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v55
+; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v0
+; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
-; GFX950-NEXT:    v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
+; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v54
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v41
-; GFX950-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v16
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
+; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v31
+; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v31
+; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
+; GFX950-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
-; GFX950-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v55
-; GFX950-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v42
-; GFX950-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v40
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
+; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v52
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v15
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v41, v40
-; GFX950-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v52
+; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
+; GFX950-NEXT:    v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v14
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v40, v52
-; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v31
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -9374,11 +9373,11 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v13
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], 0, v13
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v52, v30
+; GFX950-NEXT:    v_cmp_gt_f32_e32 vcc, v31, v30
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
@@ -9634,8 +9633,9 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_perm_b32 v11, v35, v11, s0
 ; GFX950-NEXT:    v_perm_b32 v12, v34, v12, s0
 ; GFX950-NEXT:    v_perm_b32 v13, v32, v13, s0
-; GFX950-NEXT:    v_perm_b32 v14, v31, v14, s0
+; GFX950-NEXT:    v_perm_b32 v14, v42, v14, s0
 ; GFX950-NEXT:    v_perm_b32 v15, v33, v15, s0
+; GFX950-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximumnum_v32bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.ll
index 832c127622751..edaacdffcc8e5 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.ll
@@ -6437,52 +6437,63 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX7-LABEL: v_maximumnum_v32f16:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT:    v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT:    v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT:    v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT:    v_max_f32_e32 v31, v32, v31
-; GFX7-NEXT:    v_max_f32_e32 v33, v35, v33
-; GFX7-NEXT:    v_max_f32_e32 v32, v37, v36
-; GFX7-NEXT:    v_max_f32_e32 v35, v39, v38
-; GFX7-NEXT:    v_max_f32_e32 v36, v49, v48
-; GFX7-NEXT:    v_max_f32_e32 v37, v51, v50
-; GFX7-NEXT:    v_max_f32_e32 v39, v53, v52
-; GFX7-NEXT:    v_max_f32_e32 v48, v55, v54
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT:    v_max_f32_e32 v31, v33, v32
+; GFX7-NEXT:    v_max_f32_e32 v33, v36, v35
+; GFX7-NEXT:    v_max_f32_e32 v32, v38, v37
+; GFX7-NEXT:    v_max_f32_e32 v35, v48, v39
+; GFX7-NEXT:    v_max_f32_e32 v36, v50, v49
+; GFX7-NEXT:    v_max_f32_e32 v37, v52, v51
+; GFX7-NEXT:    v_max_f32_e32 v39, v54, v53
+; GFX7-NEXT:    v_max_f32_e32 v48, v40, v55
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
@@ -6504,46 +6515,37 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
 ; GFX7-NEXT:    v_max_f32_e32 v53, v54, v53
-; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_max_f32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT:    v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT:    v_max_f32_e32 v49, v41, v40
+; GFX7-NEXT:    v_max_f32_e32 v49, v42, v41
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v34
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX7-NEXT:    v_max_f32_e32 v3, v3, v19
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v42
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_max_f32_e32 v40, v44, v43
+; GFX7-NEXT:    v_max_f32_e32 v40, v45, v44
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_max_f32_e32 v38, v42, v38
+; GFX7-NEXT:    v_max_f32_e32 v38, v43, v38
 ; GFX7-NEXT:    v_max_f32_e32 v54, v41, v55
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
@@ -6647,96 +6649,78 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX8-LABEL: v_maximumnum_v32f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT:    v_max_f16_sdwa v38, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v39, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v48, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v49, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v50, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v51, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v40, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v41, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v58, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v59, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_e32 v17, v17, v17
-; GFX8-NEXT:    v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT:    v_max_f16_sdwa v52, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v53, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v54, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v55, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v42, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v43, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v44, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v45, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v46, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v47, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v56, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v57, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v38, v39, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v39, v49, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v48, v51, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v51, v41, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v40, v59, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_e32 v1, v1, v17
-; GFX8-NEXT:    v_max_f16_sdwa v49, v53, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v50, v55, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v52, v43, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v53, v45, v44 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v54, v47, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v55, v57, v56 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v40
-; GFX8-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX8-NEXT:    v_max_f16_sdwa v32, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX8-NEXT:    v_max_f16_sdwa v33, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v34, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v35, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v36, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v37, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v32, v33, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v33, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v34, v35, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v35, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v36, v37, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v37, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_e32 v15, v15, v15
-; GFX8-NEXT:    v_max_f16_sdwa v33, v35, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v40, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v39, v33, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v33, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v40, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v50, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v33, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v40, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v40, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v32, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v34, v15, v15
+; GFX8-NEXT:    v_max_f16_sdwa v15, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v51, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v54, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GFX8-NEXT:    v_max_f16_sdwa v55, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v28, v28, v28
+; GFX8-NEXT:    v_max_f16_e32 v12, v12, v12
+; GFX8-NEXT:    v_max_f16_e32 v24, v24, v24
+; GFX8-NEXT:    v_max_f16_e32 v8, v8, v8
+; GFX8-NEXT:    v_max_f16_sdwa v38, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v48, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v53, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v41, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v42, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_e32 v12, v12, v28
+; GFX8-NEXT:    v_max_f16_e32 v8, v8, v24
+; GFX8-NEXT:    v_max_f16_sdwa v38, v48, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v48, v41, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v8, v8, v42
+; GFX8-NEXT:    v_or_b32_e32 v12, v12, v40
+; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX8-NEXT:    v_max_f16_sdwa v15, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v35, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v36, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v52, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v37, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v52, v52, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v49, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v15, v49, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_max_f16_e32 v30, v30, v30
 ; GFX8-NEXT:    v_max_f16_e32 v14, v14, v14
 ; GFX8-NEXT:    v_max_f16_e32 v29, v29, v29
 ; GFX8-NEXT:    v_max_f16_e32 v13, v13, v13
-; GFX8-NEXT:    v_max_f16_e32 v28, v28, v28
-; GFX8-NEXT:    v_max_f16_e32 v12, v12, v12
 ; GFX8-NEXT:    v_max_f16_e32 v27, v27, v27
 ; GFX8-NEXT:    v_max_f16_e32 v11, v11, v11
 ; GFX8-NEXT:    v_max_f16_e32 v26, v26, v26
 ; GFX8-NEXT:    v_max_f16_e32 v10, v10, v10
 ; GFX8-NEXT:    v_max_f16_e32 v25, v25, v25
 ; GFX8-NEXT:    v_max_f16_e32 v9, v9, v9
-; GFX8-NEXT:    v_max_f16_e32 v24, v24, v24
-; GFX8-NEXT:    v_max_f16_e32 v8, v8, v8
 ; GFX8-NEXT:    v_max_f16_e32 v23, v23, v23
 ; GFX8-NEXT:    v_max_f16_e32 v7, v7, v7
 ; GFX8-NEXT:    v_max_f16_e32 v22, v22, v22
@@ -6749,42 +6733,43 @@ define <32 x half> @v_maximumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX8-NEXT:    v_max_f16_e32 v3, v3, v3
 ; GFX8-NEXT:    v_max_f16_e32 v18, v18, v18
 ; GFX8-NEXT:    v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT:    v_max_f16_e32 v17, v17, v17
+; GFX8-NEXT:    v_max_f16_e32 v1, v1, v1
 ; GFX8-NEXT:    v_max_f16_e32 v16, v16, v16
 ; GFX8-NEXT:    v_max_f16_e32 v0, v0, v0
 ; GFX8-NEXT:    v_max_f16_e32 v14, v14, v30
 ; GFX8-NEXT:    v_max_f16_e32 v13, v13, v29
-; GFX8-NEXT:    v_max_f16_e32 v12, v12, v28
 ; GFX8-NEXT:    v_max_f16_e32 v11, v11, v27
 ; GFX8-NEXT:    v_max_f16_e32 v10, v10, v26
 ; GFX8-NEXT:    v_max_f16_e32 v9, v9, v25
-; GFX8-NEXT:    v_max_f16_e32 v8, v8, v24
 ; GFX8-NEXT:    v_max_f16_e32 v7, v7, v23
 ; GFX8-NEXT:    v_max_f16_e32 v6, v6, v22
 ; GFX8-NEXT:    v_max_f16_e32 v5, v5, v21
 ; GFX8-NEXT:    v_max_f16_e32 v4, v4, v20
 ; GFX8-NEXT:    v_max_f16_e32 v3, v3, v19
 ; GFX8-NEXT:    v_max_f16_e32 v2, v2, v18
+; GFX8-NEXT:    v_max_f16_e32 v1, v1, v17
 ; GFX8-NEXT:    v_max_f16_e32 v0, v0, v16
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v33
-; GFX8-NEXT:    v_or_b32_e32 v2, v2, v55
-; GFX8-NEXT:    v_or_b32_e32 v3, v3, v54
-; GFX8-NEXT:    v_or_b32_e32 v4, v4, v53
-; GFX8-NEXT:    v_or_b32_e32 v5, v5, v52
-; GFX8-NEXT:    v_or_b32_e32 v6, v6, v51
-; GFX8-NEXT:    v_or_b32_e32 v7, v7, v50
-; GFX8-NEXT:    v_or_b32_e32 v8, v8, v49
-; GFX8-NEXT:    v_or_b32_e32 v9, v9, v48
-; GFX8-NEXT:    v_or_b32_e32 v10, v10, v39
-; GFX8-NEXT:    v_or_b32_e32 v11, v11, v38
-; GFX8-NEXT:    v_or_b32_e32 v12, v12, v36
-; GFX8-NEXT:    v_or_b32_e32 v13, v13, v34
-; GFX8-NEXT:    v_or_b32_e32 v14, v14, v32
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_max_f16_sdwa v35, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v48
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v38
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v15
+; GFX8-NEXT:    s_waitcnt vmcnt(3)
+; GFX8-NEXT:    v_max_f16_sdwa v49, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX8-NEXT:    v_max_f16_e32 v31, v31, v31
-; GFX8-NEXT:    v_max_f16_sdwa v35, v37, v35 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_e32 v15, v15, v31
-; GFX8-NEXT:    v_or_b32_e32 v15, v15, v35
+; GFX8-NEXT:    v_max_f16_sdwa v32, v32, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_e32 v31, v34, v31
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v33
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v52
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v37
+; GFX8-NEXT:    v_or_b32_e32 v6, v6, v36
+; GFX8-NEXT:    v_or_b32_e32 v7, v7, v35
+; GFX8-NEXT:    v_or_b32_e32 v9, v9, v55
+; GFX8-NEXT:    v_or_b32_e32 v10, v10, v54
+; GFX8-NEXT:    v_or_b32_e32 v11, v11, v51
+; GFX8-NEXT:    v_or_b32_e32 v13, v13, v50
+; GFX8-NEXT:    v_or_b32_e32 v14, v14, v39
+; GFX8-NEXT:    v_or_b32_e32 v15, v31, v32
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_maximumnum_v32f16:
@@ -10519,52 +10504,63 @@ define <32 x half> @v_maximumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
 ; GFX7-LABEL: v_maximumnum_v32f16_no_ieee:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT:    v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT:    v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT:    v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT:    v_max_f32_e32 v31, v32, v31
-; GFX7-NEXT:    v_max_f32_e32 v33, v35, v33
-; GFX7-NEXT:    v_max_f32_e32 v32, v37, v36
-; GFX7-NEXT:    v_max_f32_e32 v35, v39, v38
-; GFX7-NEXT:    v_max_f32_e32 v36, v49, v48
-; GFX7-NEXT:    v_max_f32_e32 v37, v51, v50
-; GFX7-NEXT:    v_max_f32_e32 v39, v53, v52
-; GFX7-NEXT:    v_max_f32_e32 v48, v55, v54
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT:    v_max_f32_e32 v31, v33, v32
+; GFX7-NEXT:    v_max_f32_e32 v33, v36, v35
+; GFX7-NEXT:    v_max_f32_e32 v32, v38, v37
+; GFX7-NEXT:    v_max_f32_e32 v35, v48, v39
+; GFX7-NEXT:    v_max_f32_e32 v36, v50, v49
+; GFX7-NEXT:    v_max_f32_e32 v37, v52, v51
+; GFX7-NEXT:    v_max_f32_e32 v39, v54, v53
+; GFX7-NEXT:    v_max_f32_e32 v48, v40, v55
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
@@ -10586,46 +10582,37 @@ define <32 x half> @v_maximumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
 ; GFX7-NEXT:    v_max_f32_e32 v53, v54, v53
-; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_max_f32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT:    v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT:    v_max_f32_e32 v49, v41, v40
+; GFX7-NEXT:    v_max_f32_e32 v49, v42, v41
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v34
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX7-NEXT:    v_max_f32_e32 v3, v3, v19
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v42
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_max_f32_e32 v40, v44, v43
+; GFX7-NEXT:    v_max_f32_e32 v40, v45, v44
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_max_f32_e32 v38, v42, v38
+; GFX7-NEXT:    v_max_f32_e32 v38, v43, v38
 ; GFX7-NEXT:    v_max_f32_e32 v54, v41, v55
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index d5b2e770283c5..fc200a2ec541f 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -7991,7 +7991,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_cmp_eq_f32_e64 s[8:9], 0, v38
 ; GFX8-NEXT:    s_and_b64 vcc, s[6:7], vcc
 ; GFX8-NEXT:    v_and_b32_e32 v33, 0xffff0000, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v42, v32, v31, vcc
 ; GFX8-NEXT:    s_and_b64 vcc, s[8:9], s[4:5]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
 ; GFX8-NEXT:    v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -8005,9 +8005,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
 ; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX8-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX8-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v35, 16, v52
 ; GFX8-NEXT:    v_and_b32_e32 v36, 0xffff0000, v52
@@ -8189,76 +8186,79 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX8-NEXT:    v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v53
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v55, v40
+; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v18
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v53
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v31, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v53
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX8-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v41
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v54
-; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
-; GFX8-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX8-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX8-NEXT:    v_and_b32_e32 v55, 0xffff0000, v17
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX8-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v42
-; GFX8-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v40
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v55
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v31
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX8-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX8-NEXT:    v_lshrrev_b32_e32 v40, 16, v0
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX8-NEXT:    v_and_b32_e32 v40, 0xffff0000, v16
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v41, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
-; GFX8-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
+; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v31
+; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v52
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX8-NEXT:    v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v15
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
+; GFX8-NEXT:    v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
+; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
 ; GFX8-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX8-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v31
 ; GFX8-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v14
 ; GFX8-NEXT:    s_and_b64 vcc, vcc, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8269,8 +8269,8 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX8-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
-; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v30
+; GFX8-NEXT:    v_lshlrev_b32_e32 v31, 16, v13
+; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v31, v30
 ; GFX8-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8377,9 +8377,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v24, 16, v6
 ; GFX8-NEXT:    v_cmp_lt_f32_e32 vcc, v24, v23
 ; GFX8-NEXT:    v_cndmask_b32_e32 v22, v22, v6, vcc
-; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v23, 16, v22
 ; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v6
@@ -8503,7 +8500,10 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX8-NEXT:    v_or_b32_sdwa v12, v12, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v32
 ; GFX8-NEXT:    v_or_b32_sdwa v13, v13, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v31
+; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v42
+; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX8-NEXT:    v_or_b32_sdwa v14, v14, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v16, 16, v33
 ; GFX8-NEXT:    v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -8550,7 +8550,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_cmp_eq_f32_e64 s[8:9], 0, v38
 ; GFX900-NEXT:    s_and_b64 vcc, s[6:7], vcc
 ; GFX900-NEXT:    v_and_b32_e32 v33, 0xffff0000, v15
-; GFX900-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v42, v32, v31, vcc
 ; GFX900-NEXT:    s_and_b64 vcc, s[8:9], s[4:5]
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
 ; GFX900-NEXT:    v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -8564,9 +8564,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
 ; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX900-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX900-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v35, 16, v52
 ; GFX900-NEXT:    v_and_b32_e32 v36, 0xffff0000, v52
@@ -8748,76 +8745,79 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v2
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX900-NEXT:    v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v53
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v55, v40
+; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v18
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
 ; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v53
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v31, v55
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v53
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX900-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v41
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
-; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v54
-; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
-; GFX900-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX900-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
+; GFX900-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX900-NEXT:    v_and_b32_e32 v55, 0xffff0000, v17
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX900-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
-; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v42
-; GFX900-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v40
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v55
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v55
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
+; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v31
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX900-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX900-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX900-NEXT:    v_lshrrev_b32_e32 v40, 16, v0
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX900-NEXT:    v_and_b32_e32 v40, 0xffff0000, v16
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v41, 16, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
-; GFX900-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
+; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v31
+; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; GFX900-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v52
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
+; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX900-NEXT:    v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v15
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
+; GFX900-NEXT:    v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
+; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
 ; GFX900-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
-; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
+; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v31
 ; GFX900-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v14
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
 ; GFX900-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -8828,8 +8828,8 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX900-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
-; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v31, 16, v13
+; GFX900-NEXT:    v_cmp_lt_f32_e32 vcc, v31, v30
 ; GFX900-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v30
@@ -8914,9 +8914,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v7
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v24, 16, v23
-; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
 ; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v24, v24
 ; GFX900-NEXT:    v_cndmask_b32_e32 v23, v23, v7, vcc
@@ -9033,8 +9030,12 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
 ; GFX900-NEXT:    v_cmp_eq_u16_e64 s[4:5], s10, v0
 ; GFX900-NEXT:    s_and_b64 vcc, vcc, s[4:5]
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
 ; GFX900-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX900-NEXT:    v_perm_b32 v14, v42, v14, s4
+; GFX900-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX900-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
 ; GFX900-NEXT:    v_perm_b32 v0, v55, v0, s4
 ; GFX900-NEXT:    v_perm_b32 v1, v54, v1, s4
 ; GFX900-NEXT:    v_perm_b32 v2, v53, v2, s4
@@ -9049,7 +9050,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX900-NEXT:    v_perm_b32 v11, v35, v11, s4
 ; GFX900-NEXT:    v_perm_b32 v12, v34, v12, s4
 ; GFX900-NEXT:    v_perm_b32 v13, v32, v13, s4
-; GFX900-NEXT:    v_perm_b32 v14, v31, v14, s4
 ; GFX900-NEXT:    v_perm_b32 v15, v33, v15, s4
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
@@ -9094,7 +9094,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_cmp_eq_f32_e64 s[4:5], 0, v38
 ; GFX950-NEXT:    s_and_b64 vcc, s[2:3], vcc
 ; GFX950-NEXT:    v_and_b32_e32 v33, 0xffff0000, v15
-; GFX950-NEXT:    v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v42, v32, v31, vcc
 ; GFX950-NEXT:    s_and_b64 vcc, s[4:5], s[0:1]
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v34, 16, v15
 ; GFX950-NEXT:    v_cndmask_b32_e32 v32, v36, v35, vcc
@@ -9108,9 +9108,6 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_and_b32_e32 v53, 0xffff0000, v21
 ; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v20
 ; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v19
-; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v18
-; GFX950-NEXT:    v_and_b32_e32 v41, 0xffff0000, v17
-; GFX950-NEXT:    v_and_b32_e32 v42, 0xffff0000, v16
 ; GFX950-NEXT:    s_waitcnt vmcnt(0)
 ; GFX950-NEXT:    v_lshrrev_b32_e32 v35, 16, v52
 ; GFX950-NEXT:    v_and_b32_e32 v36, 0xffff0000, v52
@@ -9313,91 +9310,93 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v53, v53
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v53, v55, v54, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v53
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v18
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v53
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v53
 ; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v55, v40
-; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v1
+; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v31, v55
+; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v1
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v53, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v54, v53, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v54, 16, v31
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v54
+; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
+; GFX950-NEXT:    v_lshrrev_b32_e32 v54, 16, v17
+; GFX950-NEXT:    v_cndmask_b32_e32 v53, v31, v53, vcc
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v1
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    s_nop 1
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v55, v54, vcc
+; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v17
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v31
+; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v31
+; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v55
+; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v0
+; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v55, 16, v54
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v55
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v17
-; GFX950-NEXT:    v_cndmask_b32_e32 v53, v54, v53, vcc
-; GFX950-NEXT:    v_and_b32_e32 v54, 0xffff0000, v1
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v54, v54
+; GFX950-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX950-NEXT:    v_cndmask_b32_e32 v54, v54, v31, vcc
+; GFX950-NEXT:    v_and_b32_e32 v31, 0xffff0000, v0
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
 ; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v54, v40, v55, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v41, v41
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v54
-; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v54
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v41
-; GFX950-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v40, v55, vcc
+; GFX950-NEXT:    v_and_b32_e32 v40, 0xffff0000, v16
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
+; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v31
+; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v31
+; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
+; GFX950-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v54, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v55
 ; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
-; GFX950-NEXT:    v_lshrrev_b32_e32 v40, 16, v16
-; GFX950-NEXT:    v_cndmask_b32_e32 v54, v55, v54, vcc
-; GFX950-NEXT:    v_and_b32_e32 v55, 0xffff0000, v0
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v55, v55
-; GFX950-NEXT:    s_nop 1
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v41, v40, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v42, v42
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v55
-; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v55
-; GFX950-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v42, 16, v40
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v42
-; GFX950-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v40, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v40
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v41
-; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v55, v40, v55, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
+; GFX950-NEXT:    v_cndmask_b32_e32 v55, v55, v31, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v15
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v52
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v15, v15, v52, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v40, v40
-; GFX950-NEXT:    v_lshlrev_b32_e32 v41, 16, v15
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v15
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v15
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v41, v40
-; GFX950-NEXT:    v_accvgpr_read_b32 v41, a1 ; Reload Reuse
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v52, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v52
+; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v52, v52, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v52
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v40
+; GFX950-NEXT:    v_cndmask_b32_e32 v31, v31, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v31
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v15, v52, v15, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
+; GFX950-NEXT:    v_cndmask_b32_e32 v15, v31, v15, vcc
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v14
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v14, v30, vcc
-; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v52, v52
-; GFX950-NEXT:    v_lshlrev_b32_e32 v40, 16, v14
+; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v31, v31
+; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v14
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v14
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v40, v52
-; GFX950-NEXT:    v_accvgpr_read_b32 v40, a0 ; Reload Reuse
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v31
+; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v30, v30, v14, vcc
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v30
-; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v52
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v30
+; GFX950-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
 ; GFX950-NEXT:    s_and_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
@@ -9407,11 +9406,11 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    s_nop 0
 ; GFX950-NEXT:    v_cndmask_b32_e32 v13, v13, v29, vcc
 ; GFX950-NEXT:    v_cmp_u_f32_e32 vcc, v30, v30
-; GFX950-NEXT:    v_lshlrev_b32_e32 v52, 16, v13
+; GFX950-NEXT:    v_lshlrev_b32_e32 v31, 16, v13
 ; GFX950-NEXT:    v_cmp_eq_u16_e64 s[0:1], s6, v13
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
-; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v52, v30
+; GFX950-NEXT:    v_cmp_lt_f32_e32 vcc, v31, v30
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v29, v29, v13, vcc
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v30, 16, v29
@@ -9667,8 +9666,9 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
 ; GFX950-NEXT:    v_perm_b32 v11, v35, v11, s0
 ; GFX950-NEXT:    v_perm_b32 v12, v34, v12, s0
 ; GFX950-NEXT:    v_perm_b32 v13, v32, v13, s0
-; GFX950-NEXT:    v_perm_b32 v14, v31, v14, s0
+; GFX950-NEXT:    v_perm_b32 v14, v42, v14, s0
 ; GFX950-NEXT:    v_perm_b32 v15, v33, v15, s0
+; GFX950-NEXT:    v_accvgpr_read_b32 v42, a2 ; Reload Reuse
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimumnum_v32bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.ll
index cb80101095780..83ba18e515433 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.ll
@@ -6218,52 +6218,63 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX7-LABEL: v_minimumnum_v32f16:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT:    v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT:    v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT:    v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT:    v_min_f32_e32 v31, v32, v31
-; GFX7-NEXT:    v_min_f32_e32 v33, v35, v33
-; GFX7-NEXT:    v_min_f32_e32 v32, v37, v36
-; GFX7-NEXT:    v_min_f32_e32 v35, v39, v38
-; GFX7-NEXT:    v_min_f32_e32 v36, v49, v48
-; GFX7-NEXT:    v_min_f32_e32 v37, v51, v50
-; GFX7-NEXT:    v_min_f32_e32 v39, v53, v52
-; GFX7-NEXT:    v_min_f32_e32 v48, v55, v54
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT:    v_min_f32_e32 v31, v33, v32
+; GFX7-NEXT:    v_min_f32_e32 v33, v36, v35
+; GFX7-NEXT:    v_min_f32_e32 v32, v38, v37
+; GFX7-NEXT:    v_min_f32_e32 v35, v48, v39
+; GFX7-NEXT:    v_min_f32_e32 v36, v50, v49
+; GFX7-NEXT:    v_min_f32_e32 v37, v52, v51
+; GFX7-NEXT:    v_min_f32_e32 v39, v54, v53
+; GFX7-NEXT:    v_min_f32_e32 v48, v40, v55
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
@@ -6285,46 +6296,37 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
 ; GFX7-NEXT:    v_min_f32_e32 v53, v54, v53
-; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_min_f32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT:    v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT:    v_min_f32_e32 v49, v41, v40
+; GFX7-NEXT:    v_min_f32_e32 v49, v42, v41
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v34
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX7-NEXT:    v_min_f32_e32 v3, v3, v19
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v42
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_min_f32_e32 v40, v44, v43
+; GFX7-NEXT:    v_min_f32_e32 v40, v45, v44
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_min_f32_e32 v38, v42, v38
+; GFX7-NEXT:    v_min_f32_e32 v38, v43, v38
 ; GFX7-NEXT:    v_min_f32_e32 v54, v41, v55
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
@@ -6428,96 +6430,78 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX8-LABEL: v_minimumnum_v32f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX8-NEXT:    buffer_store_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT:    v_max_f16_sdwa v38, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v39, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v48, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v49, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v50, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v51, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v40, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v41, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v58, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v59, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_e32 v17, v17, v17
-; GFX8-NEXT:    v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT:    v_max_f16_sdwa v52, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v53, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v54, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v55, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v42, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v43, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v44, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v45, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v46, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v47, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v56, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v57, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_min_f16_sdwa v38, v39, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v39, v49, v48 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v48, v51, v50 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v51, v41, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v40, v59, v58 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_e32 v1, v1, v17
-; GFX8-NEXT:    v_min_f16_sdwa v49, v53, v52 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v50, v55, v54 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v52, v43, v42 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v53, v45, v44 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v54, v47, v46 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_sdwa v55, v57, v56 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_e32 v1, v1, v40
-; GFX8-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
-; GFX8-NEXT:    v_max_f16_sdwa v32, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX8-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX8-NEXT:    v_max_f16_sdwa v33, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v34, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v35, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v36, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_sdwa v37, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_min_f16_sdwa v32, v33, v32 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v33, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_min_f16_sdwa v34, v35, v34 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v35, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_min_f16_sdwa v36, v37, v36 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_max_f16_sdwa v37, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT:    v_max_f16_e32 v15, v15, v15
-; GFX8-NEXT:    v_min_f16_sdwa v33, v35, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v40, v30, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v39, v33, v40 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v33, v29, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v40, v13, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v50, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v33, v28, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v40, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v40, v40, v33 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v32, v15, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v34, v15, v15
+; GFX8-NEXT:    v_max_f16_sdwa v15, v27, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v11, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v51, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v26, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v54, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v25, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; GFX8-NEXT:    v_min_f16_sdwa v55, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v24, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v28, v28, v28
+; GFX8-NEXT:    v_max_f16_e32 v12, v12, v12
+; GFX8-NEXT:    v_max_f16_e32 v24, v24, v24
+; GFX8-NEXT:    v_max_f16_e32 v8, v8, v8
+; GFX8-NEXT:    v_max_f16_sdwa v38, v17, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v48, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v53, v16, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v41, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v42, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_e32 v12, v12, v28
+; GFX8-NEXT:    v_min_f16_e32 v8, v8, v24
+; GFX8-NEXT:    v_min_f16_sdwa v38, v48, v38 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_sdwa v48, v41, v53 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v8, v8, v42
+; GFX8-NEXT:    v_or_b32_e32 v12, v12, v40
+; GFX8-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX8-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX8-NEXT:    v_max_f16_sdwa v15, v23, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v35, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v22, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v36, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v21, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v52, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v37, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v20, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v52, v52, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v19, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v33, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v49, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v33, v33, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v15, v18, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v15, v49, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_max_f16_e32 v30, v30, v30
 ; GFX8-NEXT:    v_max_f16_e32 v14, v14, v14
 ; GFX8-NEXT:    v_max_f16_e32 v29, v29, v29
 ; GFX8-NEXT:    v_max_f16_e32 v13, v13, v13
-; GFX8-NEXT:    v_max_f16_e32 v28, v28, v28
-; GFX8-NEXT:    v_max_f16_e32 v12, v12, v12
 ; GFX8-NEXT:    v_max_f16_e32 v27, v27, v27
 ; GFX8-NEXT:    v_max_f16_e32 v11, v11, v11
 ; GFX8-NEXT:    v_max_f16_e32 v26, v26, v26
 ; GFX8-NEXT:    v_max_f16_e32 v10, v10, v10
 ; GFX8-NEXT:    v_max_f16_e32 v25, v25, v25
 ; GFX8-NEXT:    v_max_f16_e32 v9, v9, v9
-; GFX8-NEXT:    v_max_f16_e32 v24, v24, v24
-; GFX8-NEXT:    v_max_f16_e32 v8, v8, v8
 ; GFX8-NEXT:    v_max_f16_e32 v23, v23, v23
 ; GFX8-NEXT:    v_max_f16_e32 v7, v7, v7
 ; GFX8-NEXT:    v_max_f16_e32 v22, v22, v22
@@ -6530,42 +6514,43 @@ define <32 x half> @v_minimumnum_v32f16(<32 x half> %x, <32 x half> %y) #1 {
 ; GFX8-NEXT:    v_max_f16_e32 v3, v3, v3
 ; GFX8-NEXT:    v_max_f16_e32 v18, v18, v18
 ; GFX8-NEXT:    v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT:    v_max_f16_e32 v17, v17, v17
+; GFX8-NEXT:    v_max_f16_e32 v1, v1, v1
 ; GFX8-NEXT:    v_max_f16_e32 v16, v16, v16
 ; GFX8-NEXT:    v_max_f16_e32 v0, v0, v0
 ; GFX8-NEXT:    v_min_f16_e32 v14, v14, v30
 ; GFX8-NEXT:    v_min_f16_e32 v13, v13, v29
-; GFX8-NEXT:    v_min_f16_e32 v12, v12, v28
 ; GFX8-NEXT:    v_min_f16_e32 v11, v11, v27
 ; GFX8-NEXT:    v_min_f16_e32 v10, v10, v26
 ; GFX8-NEXT:    v_min_f16_e32 v9, v9, v25
-; GFX8-NEXT:    v_min_f16_e32 v8, v8, v24
 ; GFX8-NEXT:    v_min_f16_e32 v7, v7, v23
 ; GFX8-NEXT:    v_min_f16_e32 v6, v6, v22
 ; GFX8-NEXT:    v_min_f16_e32 v5, v5, v21
 ; GFX8-NEXT:    v_min_f16_e32 v4, v4, v20
 ; GFX8-NEXT:    v_min_f16_e32 v3, v3, v19
 ; GFX8-NEXT:    v_min_f16_e32 v2, v2, v18
+; GFX8-NEXT:    v_min_f16_e32 v1, v1, v17
 ; GFX8-NEXT:    v_min_f16_e32 v0, v0, v16
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v33
-; GFX8-NEXT:    v_or_b32_e32 v2, v2, v55
-; GFX8-NEXT:    v_or_b32_e32 v3, v3, v54
-; GFX8-NEXT:    v_or_b32_e32 v4, v4, v53
-; GFX8-NEXT:    v_or_b32_e32 v5, v5, v52
-; GFX8-NEXT:    v_or_b32_e32 v6, v6, v51
-; GFX8-NEXT:    v_or_b32_e32 v7, v7, v50
-; GFX8-NEXT:    v_or_b32_e32 v8, v8, v49
-; GFX8-NEXT:    v_or_b32_e32 v9, v9, v48
-; GFX8-NEXT:    v_or_b32_e32 v10, v10, v39
-; GFX8-NEXT:    v_or_b32_e32 v11, v11, v38
-; GFX8-NEXT:    v_or_b32_e32 v12, v12, v36
-; GFX8-NEXT:    v_or_b32_e32 v13, v13, v34
-; GFX8-NEXT:    v_or_b32_e32 v14, v14, v32
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_max_f16_sdwa v35, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v48
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v38
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v15
+; GFX8-NEXT:    s_waitcnt vmcnt(3)
+; GFX8-NEXT:    v_max_f16_sdwa v49, v31, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX8-NEXT:    v_max_f16_e32 v31, v31, v31
-; GFX8-NEXT:    v_min_f16_sdwa v35, v37, v35 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT:    v_min_f16_e32 v15, v15, v31
-; GFX8-NEXT:    v_or_b32_e32 v15, v15, v35
+; GFX8-NEXT:    v_min_f16_sdwa v32, v32, v49 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_e32 v31, v34, v31
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v33
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v52
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v37
+; GFX8-NEXT:    v_or_b32_e32 v6, v6, v36
+; GFX8-NEXT:    v_or_b32_e32 v7, v7, v35
+; GFX8-NEXT:    v_or_b32_e32 v9, v9, v55
+; GFX8-NEXT:    v_or_b32_e32 v10, v10, v54
+; GFX8-NEXT:    v_or_b32_e32 v11, v11, v51
+; GFX8-NEXT:    v_or_b32_e32 v13, v13, v50
+; GFX8-NEXT:    v_or_b32_e32 v14, v14, v39
+; GFX8-NEXT:    v_or_b32_e32 v15, v31, v32
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_minimumnum_v32f16:
@@ -10300,52 +10285,63 @@ define <32 x half> @v_minimumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
 ; GFX7-LABEL: v_minimumnum_v32f16_no_ieee:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX7-NEXT:    buffer_store_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v30
-; GFX7-NEXT:    v_lshrrev_b32_e32 v32, 16, v14
-; GFX7-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
-; GFX7-NEXT:    v_lshrrev_b32_e32 v35, 16, v13
-; GFX7-NEXT:    v_lshrrev_b32_e32 v36, 16, v28
-; GFX7-NEXT:    v_lshrrev_b32_e32 v37, 16, v12
-; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v27
-; GFX7-NEXT:    v_lshrrev_b32_e32 v39, 16, v11
-; GFX7-NEXT:    v_lshrrev_b32_e32 v48, 16, v26
-; GFX7-NEXT:    v_lshrrev_b32_e32 v49, 16, v10
-; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v25
-; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v9
-; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v24
-; GFX7-NEXT:    v_lshrrev_b32_e32 v53, 16, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v54, 16, v23
-; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v31, v31
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v32
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v33
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v35
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v36
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v37
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v39
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v48
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v49
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v50
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v52
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
-; GFX7-NEXT:    v_min_f32_e32 v31, v32, v31
-; GFX7-NEXT:    v_min_f32_e32 v33, v35, v33
-; GFX7-NEXT:    v_min_f32_e32 v32, v37, v36
-; GFX7-NEXT:    v_min_f32_e32 v35, v39, v38
-; GFX7-NEXT:    v_min_f32_e32 v36, v49, v48
-; GFX7-NEXT:    v_min_f32_e32 v37, v51, v50
-; GFX7-NEXT:    v_min_f32_e32 v39, v53, v52
-; GFX7-NEXT:    v_min_f32_e32 v48, v55, v54
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v32, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v14
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v33, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v29
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v35, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v36, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v28
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v37, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v27
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v39, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v48, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v26
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v49, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v10
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v50, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v25
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v51, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v52, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v24
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v53, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v8
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v23
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v15
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v21
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v31
+; GFX7-NEXT:    v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v45, v31
+; GFX7-NEXT:    v_min_f32_e32 v31, v33, v32
+; GFX7-NEXT:    v_min_f32_e32 v33, v36, v35
+; GFX7-NEXT:    v_min_f32_e32 v32, v38, v37
+; GFX7-NEXT:    v_min_f32_e32 v35, v48, v39
+; GFX7-NEXT:    v_min_f32_e32 v36, v50, v49
+; GFX7-NEXT:    v_min_f32_e32 v37, v52, v51
+; GFX7-NEXT:    v_min_f32_e32 v39, v54, v53
+; GFX7-NEXT:    v_min_f32_e32 v48, v40, v55
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v51, 16, v4
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v52, 16, v19
@@ -10367,46 +10363,37 @@ define <32 x half> @v_minimumnum_v32f16_no_ieee(<32 x half> %x, <32 x half> %y)
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v54, v54
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v40, 16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
 ; GFX7-NEXT:    v_min_f32_e32 v53, v54, v53
-; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_min_f32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v53
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v40, v40
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v43, 16, v21
-; GFX7-NEXT:    v_lshrrev_b32_e32 v44, 16, v5
-; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_lshrrev_b32_e32 v42, 16, v15
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v43, v43
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v44, v44
-; GFX7-NEXT:    v_min_f32_e32 v49, v41, v40
+; GFX7-NEXT:    v_min_f32_e32 v49, v42, v41
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v55, 16, v16
+; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v38, 16, v34
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v41, 16, v0
 ; GFX7-NEXT:    v_min_f32_e32 v3, v3, v19
 ; GFX7-NEXT:    v_or_b32_e32 v1, v1, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v51
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v42, v42
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v38, v38
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v55, v55
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v41, v41
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_min_f32_e32 v40, v44, v43
+; GFX7-NEXT:    v_min_f32_e32 v40, v45, v44
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v17, 16, v17
-; GFX7-NEXT:    v_min_f32_e32 v38, v42, v38
+; GFX7-NEXT:    v_min_f32_e32 v38, v43, v38
 ; GFX7-NEXT:    v_min_f32_e32 v54, v41, v55
 ; GFX7-NEXT:    v_or_b32_e32 v3, v3, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v40
-; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v44, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX7-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
diff --git a/llvm/test/CodeGen/AMDGPU/select.f16.ll b/llvm/test/CodeGen/AMDGPU/select.f16.ll
index f698692fc7497..b0531276b5b5c 100644
--- a/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -1264,99 +1264,99 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
 ; VI-NEXT:    buffer_load_dword v48, off, s[0:3], s32 offset:64
 ; VI-NEXT:    buffer_load_dword v49, off, s[0:3], s32 offset:56
 ; VI-NEXT:    buffer_load_dword v50, off, s[0:3], s32 offset:48
-; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:40
-; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:32
-; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:24
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:16
-; VI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:8
+; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:40
+; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:32
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:24
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:16
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:8
 ; VI-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:124
-; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:116
-; VI-NEXT:    v_lshrrev_b32_e32 v42, 16, v14
-; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v30
-; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v13
-; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v29
-; VI-NEXT:    v_lshrrev_b32_e32 v46, 16, v12
-; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v28
-; VI-NEXT:    v_lshrrev_b32_e32 v56, 16, v11
-; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v27
-; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v10
-; VI-NEXT:    v_lshrrev_b32_e32 v59, 16, v26
+; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:116
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v14
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v30
+; VI-NEXT:    v_lshrrev_b32_e32 v44, 16, v29
+; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v27
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v10
+; VI-NEXT:    v_lshrrev_b32_e32 v55, 16, v26
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v9
+; VI-NEXT:    v_lshrrev_b32_e32 v47, 16, v25
+; VI-NEXT:    v_lshrrev_b32_e32 v57, 16, v8
 ; VI-NEXT:    s_waitcnt vmcnt(14)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v31
-; VI-NEXT:    v_cndmask_b32_e32 v42, v43, v42, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v59, v52, v51, vcc
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v32
-; VI-NEXT:    v_cndmask_b32_e32 v44, v45, v44, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v13
+; VI-NEXT:    v_cndmask_b32_e32 v44, v44, v31, vcc
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v33
-; VI-NEXT:    v_cndmask_b32_e32 v46, v47, v46, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v12
+; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v28
+; VI-NEXT:    v_cndmask_b32_e32 v32, v32, v31, vcc
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v34
-; VI-NEXT:    v_cndmask_b32_e32 v34, v57, v56, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v11
+; VI-NEXT:    v_cndmask_b32_e32 v31, v43, v31, vcc
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v35
-; VI-NEXT:    v_lshrrev_b32_e32 v31, 16, v9
-; VI-NEXT:    v_lshrrev_b32_e32 v43, 16, v25
-; VI-NEXT:    v_cndmask_b32_e32 v35, v59, v58, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v34, v55, v41, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(13)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v36
-; VI-NEXT:    v_lshrrev_b32_e32 v32, 16, v8
-; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v24
-; VI-NEXT:    v_cndmask_b32_e32 v31, v43, v31, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v24
+; VI-NEXT:    v_cndmask_b32_e32 v35, v47, v53, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(12)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v37
 ; VI-NEXT:    v_lshrrev_b32_e32 v33, 16, v15
-; VI-NEXT:    v_cndmask_b32_e32 v32, v45, v32, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v36, v51, v57, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v38
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v38
 ; VI-NEXT:    s_waitcnt vmcnt(10)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v39
-; VI-NEXT:    v_cndmask_b32_e32 v33, v36, v33, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v36, 16, v7
-; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v23
+; VI-NEXT:    v_cndmask_b32_e32 v33, v37, v33, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v7
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v23
 ; VI-NEXT:    s_waitcnt vmcnt(9)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v48
-; VI-NEXT:    v_cndmask_b32_e32 v36, v37, v36, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v37, 16, v6
-; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v22
+; VI-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v6
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v22
 ; VI-NEXT:    s_waitcnt vmcnt(8)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v49
-; VI-NEXT:    v_cndmask_b32_e32 v37, v39, v37, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v39, 16, v5
-; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v21
+; VI-NEXT:    v_cndmask_b32_e32 v39, v48, v39, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v5
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v21
 ; VI-NEXT:    s_waitcnt vmcnt(7)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v50
-; VI-NEXT:    v_cndmask_b32_e32 v39, v48, v39, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v48, 16, v4
-; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v20
-; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v51
 ; VI-NEXT:    v_cndmask_b32_e32 v48, v49, v48, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v3
-; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v19
-; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v52
+; VI-NEXT:    v_lshrrev_b32_e32 v49, 16, v4
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v20
+; VI-NEXT:    s_waitcnt vmcnt(6)
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v56
 ; VI-NEXT:    v_cndmask_b32_e32 v49, v50, v49, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v2
-; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v18
-; VI-NEXT:    buffer_load_dword v52, off, s[0:3], s32 offset:108
+; VI-NEXT:    v_lshrrev_b32_e32 v50, 16, v3
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v19
+; VI-NEXT:    s_waitcnt vmcnt(5)
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v58
+; VI-NEXT:    v_cndmask_b32_e32 v50, v51, v50, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v51, 16, v2
+; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v18
+; VI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:108
 ; VI-NEXT:    buffer_load_dword v43, off, s[0:3], s32 offset:92
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v53
-; VI-NEXT:    v_cndmask_b32_e32 v50, v51, v50, vcc
-; VI-NEXT:    buffer_load_dword v51, off, s[0:3], s32 offset:100
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v45
+; VI-NEXT:    v_cndmask_b32_e32 v51, v53, v51, vcc
+; VI-NEXT:    buffer_load_dword v53, off, s[0:3], s32 offset:100
+; VI-NEXT:    v_lshrrev_b32_e32 v41, 16, v1
+; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v17
 ; VI-NEXT:    buffer_load_dword v47, off, s[0:3], s32 offset:84
 ; VI-NEXT:    buffer_load_dword v56, off, s[0:3], s32 offset:76
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v54
-; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:68
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v46
+; VI-NEXT:    v_cndmask_b32_e32 v41, v45, v41, vcc
+; VI-NEXT:    buffer_load_dword v45, off, s[0:3], s32 offset:68
 ; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:60
-; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:52
-; VI-NEXT:    v_lshrrev_b32_e32 v53, 16, v1
-; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v17
-; VI-NEXT:    v_cndmask_b32_e32 v53, v45, v53, vcc
-; VI-NEXT:    v_lshrrev_b32_e32 v45, 16, v0
+; VI-NEXT:    buffer_load_dword v46, off, s[0:3], s32 offset:52
+; VI-NEXT:    v_lshrrev_b32_e32 v52, 16, v0
 ; VI-NEXT:    v_lshrrev_b32_e32 v58, 16, v16
 ; VI-NEXT:    s_waitcnt vmcnt(10)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v55
-; VI-NEXT:    buffer_load_dword v55, off, s[0:3], s32 offset:44
-; VI-NEXT:    v_cndmask_b32_e32 v45, v58, v45, vcc
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v42
+; VI-NEXT:    buffer_load_dword v42, off, s[0:3], s32 offset:44
+; VI-NEXT:    v_cndmask_b32_e32 v52, v58, v52, vcc
 ; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:36
 ; VI-NEXT:    s_waitcnt vmcnt(11)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v40
@@ -1364,15 +1364,15 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
 ; VI-NEXT:    v_cndmask_b32_e32 v15, v38, v15, vcc
 ; VI-NEXT:    buffer_load_dword v38, off, s[0:3], s32 offset:20
 ; VI-NEXT:    s_waitcnt vmcnt(12)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v41
-; VI-NEXT:    buffer_load_dword v41, off, s[0:3], s32 offset:12
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v54
+; VI-NEXT:    buffer_load_dword v54, off, s[0:3], s32 offset:12
 ; VI-NEXT:    v_cndmask_b32_e32 v14, v30, v14, vcc
 ; VI-NEXT:    buffer_load_dword v30, off, s[0:3], s32 offset:4
 ; VI-NEXT:    s_waitcnt vmcnt(13)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v52
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v55
 ; VI-NEXT:    v_cndmask_b32_e32 v13, v29, v13, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(11)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v51
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v53
 ; VI-NEXT:    v_cndmask_b32_e32 v12, v28, v12, vcc
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v43
 ; VI-NEXT:    v_cndmask_b32_e32 v11, v27, v11, vcc
@@ -1383,16 +1383,16 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v56
 ; VI-NEXT:    v_cndmask_b32_e32 v9, v25, v9, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(8)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v54
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v45
 ; VI-NEXT:    v_cndmask_b32_e32 v8, v24, v8, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(7)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v57
 ; VI-NEXT:    v_cndmask_b32_e32 v7, v23, v7, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(6)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v59
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v46
 ; VI-NEXT:    v_cndmask_b32_e32 v6, v22, v6, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(5)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v55
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v42
 ; VI-NEXT:    v_cndmask_b32_e32 v5, v21, v5, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(4)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v58
@@ -1404,40 +1404,40 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v38
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v41
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v54
 ; VI-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v30
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v45
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v52
 ; VI-NEXT:    v_or_b32_sdwa v0, v0, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v53
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v41
 ; VI-NEXT:    v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v50
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v51
 ; VI-NEXT:    v_or_b32_sdwa v2, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v49
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v50
 ; VI-NEXT:    v_or_b32_sdwa v3, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v48
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v49
 ; VI-NEXT:    v_or_b32_sdwa v4, v4, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v39
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v48
 ; VI-NEXT:    v_or_b32_sdwa v5, v5, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v37
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v39
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v36
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v37
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v32
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v36
 ; VI-NEXT:    v_or_b32_sdwa v8, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v31
-; VI-NEXT:    v_or_b32_sdwa v9, v9, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v35
-; VI-NEXT:    v_or_b32_sdwa v10, v10, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v9, v9, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v34
+; VI-NEXT:    v_or_b32_sdwa v10, v10, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v31
 ; VI-NEXT:    v_or_b32_sdwa v11, v11, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v46
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v32
 ; VI-NEXT:    v_or_b32_sdwa v12, v12, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v44
 ; VI-NEXT:    v_or_b32_sdwa v13, v13, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v42
+; VI-NEXT:    v_lshlrev_b32_e32 v16, 16, v59
 ; VI-NEXT:    buffer_load_dword v59, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v58, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
 ; VI-NEXT:    buffer_load_dword v57, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll b/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll
index d50a149bb0407..86cd7f2d0f1a4 100644
--- a/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll
+++ b/llvm/test/CodeGen/RISCV/csr-first-use-cost.ll
@@ -48,7 +48,6 @@ define fastcc void @Perl_sv_setnv(i8 %c, ptr %.str.54.3682) nounwind {
 ; DEFAULT-COST:       # %bb.0: # %entry
 ; DEFAULT-COST-NEXT:    addi sp, sp, -32
 ; DEFAULT-COST-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; DEFAULT-COST-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; DEFAULT-COST-NEXT:    li a2, 2
 ; DEFAULT-COST-NEXT:    zext.b a0, a0
 ; DEFAULT-COST-NEXT:    blt a2, a0, .LBB0_3
@@ -64,7 +63,6 @@ define fastcc void @Perl_sv_setnv(i8 %c, ptr %.str.54.3682) nounwind {
 ; DEFAULT-COST-NEXT:    bne a0, a2, .LBB0_5
 ; DEFAULT-COST-NEXT:  .LBB0_4: # %sw.bb3
 ; DEFAULT-COST-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; DEFAULT-COST-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; DEFAULT-COST-NEXT:    addi sp, sp, 32
 ; DEFAULT-COST-NEXT:    ret
 ; DEFAULT-COST-NEXT:  .LBB0_5: # %entry
@@ -74,14 +72,16 @@ define fastcc void @Perl_sv_setnv(i8 %c, ptr %.str.54.3682) nounwind {
 ; DEFAULT-COST-NEXT:  .LBB0_6: # %sw.bb34.i
 ; DEFAULT-COST-NEXT:    sd zero, 8(sp) # 8-byte Folded Spill
 ; DEFAULT-COST-NEXT:  .LBB0_7: # %Perl_sv_reftype.exit
-; DEFAULT-COST-NEXT:    li s0, 0
-; DEFAULT-COST-NEXT:    li a0, 0
-; DEFAULT-COST-NEXT:    li a1, 0
-; DEFAULT-COST-NEXT:    jalr s0
-; DEFAULT-COST-NEXT:    li a0, 0
+; DEFAULT-COST-NEXT:    sd zero, 16(sp) # 8-byte Folded Spill
+; DEFAULT-COST-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT:    ld a1, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT:    ld a2, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT:    jalr a2
+; DEFAULT-COST-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
 ; DEFAULT-COST-NEXT:    ld a1, 8(sp) # 8-byte Folded Reload
-; DEFAULT-COST-NEXT:    li a2, 0
-; DEFAULT-COST-NEXT:    jalr s0
+; DEFAULT-COST-NEXT:    ld a2, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT:    ld a3, 16(sp) # 8-byte Folded Reload
+; DEFAULT-COST-NEXT:    jalr a3
 entry:
   switch i8 %c, label %Perl_sv_reftype.exit [
     i8 1, label %sw.bb4
diff --git a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
index c46ce8801702b..6eb2227cebab9 100644
--- a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
+++ b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
@@ -19,41 +19,37 @@ define void @caller(ptr %p) personality ptr @__gxx_personality_v0 {
 ; RV32I-NEXT:    .cfi_def_cfa_offset 16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    .cfi_offset ra, -4
 ; RV32I-NEXT:    .cfi_offset s0, -8
-; RV32I-NEXT:    .cfi_offset s1, -12
 ; RV32I-NEXT:    .cfi_remember_state
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    beqz a0, .LBB0_2
 ; RV32I-NEXT:  # %bb.1: # %bb2
-; RV32I-NEXT:  .Ltmp0:
+; RV32I-NEXT:  .Ltmp0: # EH_LABEL
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call bar
-; RV32I-NEXT:  .Ltmp1:
+; RV32I-NEXT:  .Ltmp1: # EH_LABEL
 ; RV32I-NEXT:    j .LBB0_3
 ; RV32I-NEXT:  .LBB0_2: # %bb1
-; RV32I-NEXT:  .Ltmp2:
+; RV32I-NEXT:  .Ltmp2: # EH_LABEL
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call foo
-; RV32I-NEXT:  .Ltmp3:
+; RV32I-NEXT:  .Ltmp3: # EH_LABEL
 ; RV32I-NEXT:  .LBB0_3: # %end2
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    .cfi_restore ra
 ; RV32I-NEXT:    .cfi_restore s0
-; RV32I-NEXT:    .cfi_restore s1
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_4: # %lpad
 ; RV32I-NEXT:    .cfi_restore_state
-; RV32I-NEXT:  .Ltmp4:
-; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:  .Ltmp4: # EH_LABEL
+; RV32I-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    call _Unwind_Resume
 ;
 ; RV64I-LABEL: caller:
@@ -62,41 +58,37 @@ define void @caller(ptr %p) personality ptr @__gxx_personality_v0 {
 ; RV64I-NEXT:    .cfi_def_cfa_offset 32
 ; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    .cfi_offset s0, -16
-; RV64I-NEXT:    .cfi_offset s1, -24
 ; RV64I-NEXT:    .cfi_remember_state
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    beqz a0, .LBB0_2
 ; RV64I-NEXT:  # %bb.1: # %bb2
-; RV64I-NEXT:  .Ltmp0:
+; RV64I-NEXT:  .Ltmp0: # EH_LABEL
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call bar
-; RV64I-NEXT:  .Ltmp1:
+; RV64I-NEXT:  .Ltmp1: # EH_LABEL
 ; RV64I-NEXT:    j .LBB0_3
 ; RV64I-NEXT:  .LBB0_2: # %bb1
-; RV64I-NEXT:  .Ltmp2:
+; RV64I-NEXT:  .Ltmp2: # EH_LABEL
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call foo
-; RV64I-NEXT:  .Ltmp3:
+; RV64I-NEXT:  .Ltmp3: # EH_LABEL
 ; RV64I-NEXT:  .LBB0_3: # %end2
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    .cfi_restore ra
 ; RV64I-NEXT:    .cfi_restore s0
-; RV64I-NEXT:    .cfi_restore s1
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB0_4: # %lpad
 ; RV64I-NEXT:    .cfi_restore_state
-; RV64I-NEXT:  .Ltmp4:
-; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:  .Ltmp4: # EH_LABEL
+; RV64I-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    call _Unwind_Resume
 entry:
   %0 = icmp eq ptr %p, null
diff --git a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
index e264e74fee9d3..3860a9a6818a1 100644
--- a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
+++ b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll
@@ -22,17 +22,14 @@
 define ptr @ubyte_divmod(ptr %a, ptr %b) {
 ; CHECK-LABEL: ubyte_divmod:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    subq $40, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset %rbx, -32
-; CHECK-NEXT:    .cfi_offset %r14, -24
-; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r14, -16
 ; CHECK-NEXT:    movq %rsi, %rbx
 ; CHECK-NEXT:    movq %rdi, %r14
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
@@ -107,7 +104,8 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    movzbl %sil, %eax
 ; CHECK-NEXT:    divb %dl
-; CHECK-NEXT:    movzbl %ah, %ebp
+; CHECK-NEXT:    movzbl %ah, %eax
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    jmp LBB0_18
 ; CHECK-NEXT:  LBB0_14: ## %cond_true.i200
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
@@ -117,7 +115,7 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
 ; CHECK-NEXT:    movl $4, %edi
 ; CHECK-NEXT:    callq _feraiseexcept
 ; CHECK-NEXT:  LBB0_17: ## %ubyte_ctype_remainder.exit
-; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
 ; CHECK-NEXT:  LBB0_18: ## %ubyte_ctype_remainder.exit
 ; CHECK-NEXT:    movq (%r14), %rax
 ; CHECK-NEXT:    callq *224(%rax)
@@ -168,7 +166,8 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
 ; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.24: ## %cond_next182
-; CHECK-NEXT:    movb %bpl, 16(%rax)
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx ## 4-byte Reload
+; CHECK-NEXT:    movb %cl, 16(%rax)
 ; CHECK-NEXT:    movq %rax, 32(%rbx)
 ; CHECK-NEXT:    movq %rbx, %rax
 ; CHECK-NEXT:    jmp LBB0_28
@@ -182,10 +181,9 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) {
 ; CHECK-NEXT:  LBB0_27: ## %UnifiedReturnBlock
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:  LBB0_28: ## %UnifiedReturnBlock
-; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    addq $40, %rsp
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    popq %r14
-; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    retq
 entry:
 	%arg1 = alloca i8, align 1		; <ptr> [#uses=3]
diff --git a/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll b/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
index 5901b9b8300fa..4d53a6d7873fa 100644
--- a/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
+++ b/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
@@ -12,15 +12,16 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    pushl %ebx
 ; CHECK-NEXT:    pushl %edi
 ; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    pushl %eax
-; CHECK-NEXT:    movb $1, %cl
-; CHECK-NEXT:    movl $1, %esi
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    movb $1, %bl
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb.i5
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %ecx, %eax
-; CHECK-NEXT:    addl %esi, %esi
-; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    addl %ecx, %ecx
+; CHECK-NEXT:    xorl %ebx, %ebx
 ; CHECK-NEXT:    testb $1, %al
 ; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %mp_unexp_mp2d.exit.i
@@ -29,35 +30,34 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    jne .LBB0_3
 ; CHECK-NEXT:  # %bb.6: # %cond_next36.i
 ; CHECK-NEXT:    movl $0, 0
-; CHECK-NEXT:    movzbl %al, %ebp
-; CHECK-NEXT:    andl $1, %ebp
+; CHECK-NEXT:    movzbl %al, %ebx
+; CHECK-NEXT:    andl $1, %ebx
 ; CHECK-NEXT:    xorpd %xmm0, %xmm0
+; CHECK-NEXT:    xorl %ebp, %ebp
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    xorpd %xmm1, %xmm1
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_7: # %bb.i28.i
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    cvttsd2si %xmm1, %ecx
-; CHECK-NEXT:    cmpl %edx, %ecx
-; CHECK-NEXT:    cmovgel %eax, %ecx
-; CHECK-NEXT:    addl $2, %edi
+; CHECK-NEXT:    cvttsd2si %xmm1, %esi
+; CHECK-NEXT:    cmpl %edx, %esi
+; CHECK-NEXT:    cmovgel %ebp, %esi
+; CHECK-NEXT:    addl $2, %eax
 ; CHECK-NEXT:    xorps %xmm2, %xmm2
-; CHECK-NEXT:    cvtsi2sd %ecx, %xmm2
+; CHECK-NEXT:    cvtsi2sd %esi, %xmm2
 ; CHECK-NEXT:    xorpd %xmm1, %xmm1
 ; CHECK-NEXT:    subsd %xmm2, %xmm1
 ; CHECK-NEXT:    mulsd %xmm0, %xmm1
-; CHECK-NEXT:    addl $-2, %ebp
+; CHECK-NEXT:    addl $-2, %ebx
 ; CHECK-NEXT:    jne .LBB0_7
 ; CHECK-NEXT:  # %bb.8: # %mp_unexp_d2mp.exit29.i
 ; CHECK-NEXT:    movl $0, 0
 ; CHECK-NEXT:    je .LBB0_9
 ; CHECK-NEXT:  # %bb.10: # %mp_sqrt_init.exit
-; CHECK-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; CHECK-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movl %edx, %edi
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; CHECK-NEXT:    movl %ebx, %edx
+; CHECK-NEXT:    movl %edx, (%esp) # 4-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    calll mp_mul_csqu at PLT
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    movl $-1, %edx
@@ -67,18 +67,17 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    calll rdft at PLT
 ; CHECK-NEXT:    addl $12, %esp
 ; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movl (%esp), %edx # 4-byte Reload
 ; CHECK-NEXT:    movl %edi, %esi
-; CHECK-NEXT:    movl %edi, %edx
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    pushl $0
 ; CHECK-NEXT:    calll mp_mul_d2i at PLT
 ; CHECK-NEXT:    addl $12, %esp
-; CHECK-NEXT:    testl %ebp, %ebp
+; CHECK-NEXT:    testl %ebx, %ebx
 ; CHECK-NEXT:    je .LBB0_11
 ; CHECK-NEXT:  .LBB0_3: # %cond_true.i
-; CHECK-NEXT:    addl $4, %esp
+; CHECK-NEXT:    addl $8, %esp
 ; CHECK-NEXT:  .LBB0_4: # %cond_true.i
 ; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    popl %edi
@@ -91,39 +90,39 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    jmp .LBB0_9
 ; CHECK-NEXT:  .LBB0_11: # %cond_false.i
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movl %esi, %edx
+; CHECK-NEXT:    movl (%esp), %edx # 4-byte Reload
 ; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    pushl $0
 ; CHECK-NEXT:    calll mp_round at PLT
 ; CHECK-NEXT:    addl $8, %esp
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movl %esi, %edx
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    pushl %ebx
 ; CHECK-NEXT:    calll mp_add at PLT
 ; CHECK-NEXT:    addl $12, %esp
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movl %esi, %edx
-; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    calll mp_sub at PLT
 ; CHECK-NEXT:    addl $12, %esp
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movl %esi, %edx
-; CHECK-NEXT:    pushl %ebx
+; CHECK-NEXT:    movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT:    pushl %edi
 ; CHECK-NEXT:    pushl $0
 ; CHECK-NEXT:    calll mp_round at PLT
 ; CHECK-NEXT:    addl $8, %esp
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movl %esi, %edx
-; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    movl (%esp), %edx # 4-byte Reload
+; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    pushl {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; CHECK-NEXT:    calll mp_mul_d2i at PLT
-; CHECK-NEXT:    addl $16, %esp
+; CHECK-NEXT:    addl $20, %esp
 ; CHECK-NEXT:    jmp .LBB0_4
 entry:
 	br label %bb.i5
diff --git a/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll b/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
index 6f1419c538eb7..45b701307a3fd 100644
--- a/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-16-ReMatBug.ll
@@ -19,9 +19,8 @@ define i16 @SQLDriversW(ptr %henv, i16 zeroext  %fDir, ptr %szDrvDesc, i16 signe
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    je LBB0_1
 ; CHECK-NEXT:  ## %bb.3: ## %bb28
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebp
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebp
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebx
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
@@ -36,16 +35,18 @@ define i16 @SQLDriversW(ptr %henv, i16 zeroext  %fDir, ptr %szDrvDesc, i16 signe
 ; CHECK-NEXT:    subl $12, %esp
 ; CHECK-NEXT:    movzwl %di, %eax
 ; CHECK-NEXT:    movswl %bp, %ebp
-; CHECK-NEXT:    movswl %bx, %edi
+; CHECK-NEXT:    movswl %bx, %ebx
 ; CHECK-NEXT:    pushl $87
-; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    movl %esi, %edi
+; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    pushl %ebp
 ; CHECK-NEXT:    pushl $0
-; CHECK-NEXT:    movl %esi, %ebx
-; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    pushl %edi
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    pushl %edx
+; CHECK-NEXT:    pushl %ebx
 ; CHECK-NEXT:    pushl $0
 ; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    movl %ecx, %ebx
 ; CHECK-NEXT:    pushl %ecx
 ; CHECK-NEXT:    calll _SQLDrivers_Internal
 ; CHECK-NEXT:    addl $48, %esp
@@ -53,38 +54,38 @@ define i16 @SQLDriversW(ptr %henv, i16 zeroext  %fDir, ptr %szDrvDesc, i16 signe
 ; CHECK-NEXT:    testb %cl, %cl
 ; CHECK-NEXT:    je LBB0_1
 ; CHECK-NEXT:  ## %bb.5:
-; CHECK-NEXT:    movl %ebx, %esi
-; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT:    movl %ebx, %ecx
+; CHECK-NEXT:    movl %edi, %esi
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebp
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %ebx
 ; CHECK-NEXT:  LBB0_6: ## %done
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    testb %cl, %cl
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    testb %dl, %dl
 ; CHECK-NEXT:    je LBB0_7
 ; CHECK-NEXT:  ## %bb.8: ## %bb167
 ; CHECK-NEXT:    subl $12, %esp
 ; CHECK-NEXT:    movl L_iodbcdm_global_lock$non_lazy_ptr, %ecx
 ; CHECK-NEXT:    pushl %ecx
-; CHECK-NEXT:    movl %eax, %esi
+; CHECK-NEXT:    movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
 ; CHECK-NEXT:    calll _pthread_mutex_unlock
 ; CHECK-NEXT:    addl $16, %esp
-; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 2-byte Folded Reload
 ; CHECK-NEXT:    jmp LBB0_2
 ; CHECK-NEXT:  LBB0_7: ## %bb150
 ; CHECK-NEXT:    cwtl
 ; CHECK-NEXT:    subl $8, %esp
-; CHECK-NEXT:    movswl %bp, %ecx
+; CHECK-NEXT:    movswl %bp, %edx
 ; CHECK-NEXT:    movswl %bx, %ebx
 ; CHECK-NEXT:    movzwl %di, %edi
+; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    pushl %edx
-; CHECK-NEXT:    pushl %ecx
 ; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
-; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    pushl %ebx
 ; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    pushl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    pushl %ecx
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    pushl $1
 ; CHECK-NEXT:    calll _trace_SQLDriversW
diff --git a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
index fca0ca1bbd09f..3ce244cd19d27 100644
--- a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
@@ -17,28 +17,20 @@
 define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct.wxString)  %agg.result, ptr %this, ptr %format, ptr %tz, i1 %foo) personality ptr @__gxx_personality_v0 {
 ; CHECK-LABEL: _ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    pushl %ebp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    pushl %ebx
-; CHECK-NEXT:    .cfi_def_cfa_offset 12
-; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    .cfi_def_cfa_offset 20
-; CHECK-NEXT:    subl $28, %esp
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset %esi, -20
-; CHECK-NEXT:    .cfi_offset %edi, -16
-; CHECK-NEXT:    .cfi_offset %ebx, -12
-; CHECK-NEXT:    .cfi_offset %ebp, -8
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    subl $20, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %esi, -12
+; CHECK-NEXT:    .cfi_offset %ebx, -8
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    movzbl {{[0-9]+}}(%esp), %ebx
 ; CHECK-NEXT:    testb $1, %bl
 ; CHECK-NEXT:    je LBB0_21
 ; CHECK-NEXT:  ## %bb.1:
-; CHECK-NEXT:    movl L_.str89$non_lazy_ptr, %edi
 ; CHECK-NEXT:    movb $1, %bh
-; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %ebp
 ; CHECK-NEXT:    jmp LBB0_2
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_17: ## %bb7806
@@ -56,7 +48,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct
 ; CHECK-NEXT:  ## %bb.3: ## %bb3306
 ; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:  Ltmp0: ## EH_LABEL
-; CHECK-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl L_.str89$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN12wxStringBaseaSEPKw
 ; CHECK-NEXT:  Ltmp1: ## EH_LABEL
@@ -99,7 +92,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl L__ZZNK10wxDateTime5GetTmERKNS_8TimeZoneEE12__FUNCTION__$non_lazy_ptr, %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    movl %ebp, (%esp)
+; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %eax
+; CHECK-NEXT:    movl %eax, (%esp)
 ; CHECK-NEXT:    movl $1717, {{[0-9]+}}(%esp) ## imm = 0x6B5
 ; CHECK-NEXT:    calll __Z10wxOnAssertPKwiPKcS0_S0_
 ; CHECK-NEXT:  Ltmp7: ## EH_LABEL
@@ -145,11 +139,9 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct
 ; CHECK-NEXT:  Ltmp3: ## EH_LABEL
 ; CHECK-NEXT:  LBB0_21: ## %bb115.critedge.i
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    addl $28, %esp
+; CHECK-NEXT:    addl $20, %esp
 ; CHECK-NEXT:    popl %esi
-; CHECK-NEXT:    popl %edi
 ; CHECK-NEXT:    popl %ebx
-; CHECK-NEXT:    popl %ebp
 ; CHECK-NEXT:    retl $4
 ; CHECK-NEXT:  LBB0_19: ## %lpad.loopexit.split-lp
 ; CHECK-NEXT:  Ltmp15: ## EH_LABEL
diff --git a/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll b/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll
index 283e7b3454f6c..d17824457eb2f 100644
--- a/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll
+++ b/llvm/test/CodeGen/X86/apx/memfold-origVNI-crash.ll
@@ -4,57 +4,52 @@
 define fastcc i64 @foo(ptr %p0, ptr %p1, i64 %p2, i64 %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, i64 %p9, i1 %cond, i32 %n) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    subq $56, %rsp
 ; CHECK-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    jne .LBB0_7
+; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %bb1
-; CHECK-NEXT:    jne .LBB0_7
+; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:  # %bb.4: # %bb2
 ; CHECK-NEXT:    movb $1, %r10b
 ; CHECK-NEXT:    testb %r10b, %r10b
-; CHECK-NEXT:    jne .LBB0_7
+; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:  # %bb.5: # %bb2
 ; CHECK-NEXT:    xorl %r10d, %r10d
 ; CHECK-NEXT:    testb %r10b, %r10b
-; CHECK-NEXT:    jne .LBB0_9
+; CHECK-NEXT:    jne .LBB0_8
 ; CHECK-NEXT:  # %bb.6: # %bb2
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:  .LBB0_7: # %bb3
+; CHECK-NEXT:  .LBB0_1:
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:  .LBB0_8: # %cleanup
 ; CHECK-NEXT:    addq $56, %rsp
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB0_9: # %bb5
+; CHECK-NEXT:  .LBB0_8: # %bb5
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    jne .LBB0_7
-; CHECK-NEXT:  # %bb.10: # %bb7
-; CHECK-NEXT:    jne .LBB0_7
-; CHECK-NEXT:  # %bb.12: # %bb9
-; CHECK-NEXT:    jne .LBB0_7
-; CHECK-NEXT:  # %bb.14: # %bb10
-; CHECK-NEXT:    jne .LBB0_7
-; CHECK-NEXT:  # %bb.16: # %bb11
-; CHECK-NEXT:    jne .LBB0_7
-; CHECK-NEXT:  # %bb.18: # %bb12
-; CHECK-NEXT:    je .LBB0_7
-; CHECK-NEXT:  # %bb.19: # %bb13
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.9: # %bb7
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.11: # %bb9
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.13: # %bb10
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.15: # %bb11
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.17: # %bb12
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.18: # %bb13
 ; CHECK-NEXT:    movb $1, %al
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    je .LBB0_7
-; CHECK-NEXT:  # %bb.20: # %bb15
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.19: # %bb15
 ; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
 ; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB0_7
-; CHECK-NEXT:  # %bb.21: # %bb15
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.20: # %bb15
 ; CHECK-NEXT:    cmpl $2, %eax
-; CHECK-NEXT:    je .LBB0_7
-; CHECK-NEXT:  # %bb.22: # %bb18
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.21: # %bb18
 ; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -65,27 +60,27 @@ define fastcc i64 @foo(ptr %p0, ptr %p1, i64 %p2, i64 %p3, ptr %p4, ptr %p5, ptr
 ; CHECK-NEXT:    movq %rax, (%rsp) # 8-byte Spill
 ; CHECK-NEXT:    movb $1, %al
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_27
-; CHECK-NEXT:  # %bb.23:
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %r14
-; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:  .LBB0_24: # %loop
+; CHECK-NEXT:    jne .LBB0_25
+; CHECK-NEXT:  .LBB0_22: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    callq *%rbx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    xorl %r8d, %r8d
-; CHECK-NEXT:    movq %r14, %r9
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %r9
 ; CHECK-NEXT:    pushq $0
 ; CHECK-NEXT:    pushq $0
 ; CHECK-NEXT:    pushq $0
 ; CHECK-NEXT:    pushq $0
-; CHECK-NEXT:    callq *%rbx
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    addq $32, %rsp
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
@@ -97,22 +92,23 @@ define fastcc i64 @foo(ptr %p0, ptr %p1, i64 %p2, i64 %p3, ptr %p4, ptr %p5, ptr
 ; CHECK-NEXT:    pushq $0
 ; CHECK-NEXT:    pushq $0
 ; CHECK-NEXT:    pushq $0
-; CHECK-NEXT:    callq *%rbx
+; CHECK-NEXT:    callq *%rax
 ; CHECK-NEXT:    addq $32, %rsp
 ; CHECK-NEXT:    testb $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    je .LBB0_24
-; CHECK-NEXT:  # %bb.25: # %cleanup2
-; CHECK-NEXT:    jne .LBB0_27
-; CHECK-NEXT:  # %bb.26: # %bb21
+; CHECK-NEXT:    je .LBB0_22
+; CHECK-NEXT:  # %bb.23: # %cleanup2
+; CHECK-NEXT:    jne .LBB0_25
+; CHECK-NEXT:  # %bb.24: # %bb21
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; CHECK-NEXT:    movq %rbx, %rdx
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
 ; CHECK-NEXT:    callq memcpy at PLT
-; CHECK-NEXT:    movq %rbx, (%rsp) # 8-byte Spill
-; CHECK-NEXT:  .LBB0_27: # %exit
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    movq %rax, (%rsp) # 8-byte Spill
+; CHECK-NEXT:  .LBB0_25: # %exit
 ; CHECK-NEXT:    movq (%rsp), %rax # 8-byte Reload
-; CHECK-NEXT:    jmp .LBB0_8
+; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    retq
 entry:
   br i1 %cond, label %cleanup, label %bb1
 
diff --git a/llvm/test/CodeGen/X86/atom-fixup-lea2.ll b/llvm/test/CodeGen/X86/atom-fixup-lea2.ll
index adea9e1386a90..f22d097afc802 100644
--- a/llvm/test/CodeGen/X86/atom-fixup-lea2.ll
+++ b/llvm/test/CodeGen/X86/atom-fixup-lea2.ll
@@ -1,9 +1,9 @@
 ; RUN: llc < %s -mcpu=atom -mtriple=i686-linux | FileCheck %s
 
 ; CHECK:%bb.5
+; CHECK-NEXT:addl
 ; CHECK-NEXT:leal
-; CHECK-NEXT:leal
-; CHECK-NEXT:leal
+; CHECK-NEXT:addl
 ; CHECK-NEXT:movl
 
 
diff --git a/llvm/test/CodeGen/X86/call-rv-marker.ll b/llvm/test/CodeGen/X86/call-rv-marker.ll
index 8d363bc451c63..2bb3b49c2bd51 100644
--- a/llvm/test/CodeGen/X86/call-rv-marker.ll
+++ b/llvm/test/CodeGen/X86/call-rv-marker.ll
@@ -85,14 +85,11 @@ entry:
 
 define void @rv_marker_3() personality ptr @__gxx_personality_v0 {
 ; CHECK-LABEL: rv_marker_3
-; CHECK:         pushq   %r14
+; CHECK:         pushq   %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    pushq   %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 24
-; CHECK-NEXT:    pushq   %rax
+; CHECK-NEXT:    subq    $16, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset %rbx, -24
-; CHECK-NEXT:    .cfi_offset %r14, -16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
 ; CHECK-NEXT:    callq   _foo1
 ; CHECK-NEXT:    movq    %rax, %rdi
 ; CHECK-NEXT:    callq   _objc_retainAutoreleasedReturnValue
diff --git a/llvm/test/CodeGen/X86/csr-split.ll b/llvm/test/CodeGen/X86/csr-split.ll
index 20d577efd2714..03023f3e639e5 100644
--- a/llvm/test/CodeGen/X86/csr-split.ll
+++ b/llvm/test/CodeGen/X86/csr-split.ll
@@ -76,13 +76,12 @@ define dso_local signext i32 @test2(ptr %p1) local_unnamed_addr  {
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB1_3: # %if.then2
-; X64-NEXT:    pushq %rbx
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 16
-; X64-NEXT:    .cfi_offset %rbx, -16
-; X64-NEXT:    movq %rdi, %rbx
+; X64-NEXT:    movq %rdi, (%rsp) # 8-byte Spill
 ; X64-NEXT:    callq callVoid at PLT
-; X64-NEXT:    movq %rbx, %rdi
-; X64-NEXT:    popq %rbx
+; X64-NEXT:    movq (%rsp), %rdi # 8-byte Reload
+; X64-NEXT:    popq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    jmp callNonVoid at PLT # TAILCALL
 ;
diff --git a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
index 0df3c0b03dc33..ed1cf42af3348 100644
--- a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
+++ b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
@@ -7,26 +7,14 @@
 define double @main(i224 %0) #0 {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entryitofp-entry
-; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    pushq %r15
-; CHECK-NEXT:    .cfi_def_cfa_offset 24
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    pushq %r13
-; CHECK-NEXT:    .cfi_def_cfa_offset 40
-; CHECK-NEXT:    pushq %r12
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 56
-; CHECK-NEXT:    subq $88, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    subq $120, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 144
-; CHECK-NEXT:    .cfi_offset %rbx, -56
-; CHECK-NEXT:    .cfi_offset %r12, -48
-; CHECK-NEXT:    .cfi_offset %r13, -40
-; CHECK-NEXT:    .cfi_offset %r14, -32
-; CHECK-NEXT:    .cfi_offset %r15, -24
-; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %r14, -16
 ; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    movq %rdi, %r8
 ; CHECK-NEXT:    orq %rdx, %r8
@@ -104,39 +92,45 @@ define double @main(i224 %0) #0 {
 ; CHECK-NEXT:  .LBB0_5: # %itofp-sw-default
 ; CHECK-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq %rdx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %rdx, (%rsp)
 ; CHECK-NEXT:    movl %ecx, %r8d
-; CHECK-NEXT:    movq %r8, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %r8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movb $-87, %r8b
 ; CHECK-NEXT:    subb %al, %r8b
 ; CHECK-NEXT:    movb %r8b, %bl
 ; CHECK-NEXT:    shrb $6, %bl
-; CHECK-NEXT:    movzbl %bl, %r12d
+; CHECK-NEXT:    movzbl %bl, %r14d
 ; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, (%rsp)
-; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq -24(%rsp,%r12,8), %rbx
-; CHECK-NEXT:    movq -32(%rsp,%r12,8), %r13
-; CHECK-NEXT:    movq %rcx, %rbp
+; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq 8(%rsp,%r14,8), %rbx
+; CHECK-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq (%rsp,%r14,8), %rbx
+; CHECK-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    movb %r8b, %cl
-; CHECK-NEXT:    movq %r13, %r14
-; CHECK-NEXT:    shrdq %cl, %rbx, %r14
-; CHECK-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT:    movq -48(%rsp,%r12,8), %r15
-; CHECK-NEXT:    movq -40(%rsp,%r12,8), %r12
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT:    shrdq %cl, %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    movq -16(%rsp,%r14,8), %rcx
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq -8(%rsp,%r14,8), %r14
 ; CHECK-NEXT:    movb %r8b, %cl
-; CHECK-NEXT:    movq %r12, %r14
-; CHECK-NEXT:    shrdq %cl, %r13, %r14
+; CHECK-NEXT:    movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT:    shrdq %cl, %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; CHECK-NEXT:    movb %r8b, %cl
-; CHECK-NEXT:    shrq %cl, %rbx
+; CHECK-NEXT:    shrq %cl, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; CHECK-NEXT:    movb %r8b, %cl
-; CHECK-NEXT:    shrdq %cl, %r12, %r15
+; CHECK-NEXT:    shrdq %cl, %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
 ; CHECK-NEXT:    addb $55, %al
 ; CHECK-NEXT:    movq %rdi, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rsi, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq %rbp, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rcx, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
@@ -146,17 +140,17 @@ define double @main(i224 %0) #0 {
 ; CHECK-NEXT:    andb $24, %cl
 ; CHECK-NEXT:    negb %cl
 ; CHECK-NEXT:    movsbq %cl, %rdx
-; CHECK-NEXT:    movq -80(%rsp,%rdx), %rsi
-; CHECK-NEXT:    movq -72(%rsp,%rdx), %rdi
-; CHECK-NEXT:    movq -64(%rsp,%rdx), %r8
+; CHECK-NEXT:    movq -48(%rsp,%rdx), %rsi
+; CHECK-NEXT:    movq -40(%rsp,%rdx), %rdi
+; CHECK-NEXT:    movq -32(%rsp,%rdx), %r8
 ; CHECK-NEXT:    movb %al, %cl
-; CHECK-NEXT:    movq %r8, %r12
-; CHECK-NEXT:    shldq %cl, %rdi, %r12
+; CHECK-NEXT:    movq %r8, %rbx
+; CHECK-NEXT:    shldq %cl, %rdi, %rbx
 ; CHECK-NEXT:    movb %al, %cl
-; CHECK-NEXT:    movq %rsi, %r13
-; CHECK-NEXT:    shlq %cl, %r13
-; CHECK-NEXT:    orq %r12, %r13
-; CHECK-NEXT:    movq -56(%rsp,%rdx), %rdx
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    shlq %cl, %r14
+; CHECK-NEXT:    orq %rbx, %r14
+; CHECK-NEXT:    movq -24(%rsp,%rdx), %rdx
 ; CHECK-NEXT:    movb %al, %cl
 ; CHECK-NEXT:    shldq %cl, %r8, %rdx
 ; CHECK-NEXT:    movl %edx, %edx
@@ -164,13 +158,13 @@ define double @main(i224 %0) #0 {
 ; CHECK-NEXT:    shldq %cl, %rsi, %rdi
 ; CHECK-NEXT:    orq %rdx, %rdi
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    orq %rdi, %r13
+; CHECK-NEXT:    orq %rdi, %r14
 ; CHECK-NEXT:    setne %al
-; CHECK-NEXT:    orq %rax, %r15
-; CHECK-NEXT:    movq %r15, %rdi
-; CHECK-NEXT:    movq %r14, %rsi
+; CHECK-NEXT:    orq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
 ; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; CHECK-NEXT:    movq %rbx, %rcx
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; CHECK-NEXT:    jmp .LBB0_6
 ; CHECK-NEXT:  .LBB0_6: # %itofp-sw-epilog
 ; CHECK-NEXT:    movl %edi, %eax
@@ -210,7 +204,7 @@ define double @main(i224 %0) #0 {
 ; CHECK-NEXT:    andb $24, %cl
 ; CHECK-NEXT:    negb %cl
 ; CHECK-NEXT:    movsbq %cl, %rcx
-; CHECK-NEXT:    movq 48(%rsp,%rcx), %rdx
+; CHECK-NEXT:    movq 80(%rsp,%rcx), %rdx
 ; CHECK-NEXT:    movb %al, %cl
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq %rdx, %rax
@@ -229,19 +223,11 @@ define double @main(i224 %0) #0 {
 ; CHECK-NEXT:    orq %rdx, %rax
 ; CHECK-NEXT:    movq %rax, %xmm0
 ; CHECK-NEXT:  .LBB0_10: # %itofp-return
-; CHECK-NEXT:    addq $88, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 56
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    popq %r12
-; CHECK-NEXT:    .cfi_def_cfa_offset 40
-; CHECK-NEXT:    popq %r13
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    addq $120, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 24
-; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
index 4347d626e5838..f24e0f8283bc9 100644
--- a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
@@ -43,8 +43,8 @@ define i32 @eflagsLiveInPrologue() #0 {
 ; ENABLE-NEXT:    cmovnel %edx, %ecx
 ; ENABLE-NEXT:    movl L_e$non_lazy_ptr, %edx
 ; ENABLE-NEXT:    movb %cl, (%edx)
-; ENABLE-NEXT:    leal 1(%ecx), %esi
 ; ENABLE-NEXT:    cltd
+; ENABLE-NEXT:    leal 1(%ecx), %esi
 ; ENABLE-NEXT:    idivl %esi
 ; ENABLE-NEXT:    movl L_c$non_lazy_ptr, %eax
 ; ENABLE-NEXT:    movl %edx, (%eax)
@@ -81,8 +81,8 @@ define i32 @eflagsLiveInPrologue() #0 {
 ; DISABLE-NEXT:    cmovnel %edx, %ecx
 ; DISABLE-NEXT:    movl L_e$non_lazy_ptr, %edx
 ; DISABLE-NEXT:    movb %cl, (%edx)
-; DISABLE-NEXT:    leal 1(%ecx), %esi
 ; DISABLE-NEXT:    cltd
+; DISABLE-NEXT:    leal 1(%ecx), %esi
 ; DISABLE-NEXT:    idivl %esi
 ; DISABLE-NEXT:    movl L_c$non_lazy_ptr, %eax
 ; DISABLE-NEXT:    movl %edx, (%eax)
diff --git a/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll b/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll
index b9afb99697354..92ab8af129d40 100755
--- a/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll
+++ b/llvm/test/CodeGen/X86/lrshrink-ehpad-phis.ll
@@ -26,8 +26,8 @@ define void @test() personality ptr @__gxx_personality_v0  {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 56
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
 ; CHECK-NEXT:    .cfi_offset %rbx, -56
 ; CHECK-NEXT:    .cfi_offset %r12, -48
 ; CHECK-NEXT:    .cfi_offset %r13, -40
@@ -43,12 +43,12 @@ define void @test() personality ptr @__gxx_personality_v0  {
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    movq externalC at GOTPCREL(%rip), %rax
 ; CHECK-NEXT:    movl (%rax), %eax
-; CHECK-NEXT:    movl %eax, (%rsp) # 4-byte Spill
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:  .Ltmp2: # EH_LABEL
 ; CHECK-NEXT:    callq maythrow at PLT
-; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:  .Ltmp3: # EH_LABEL
 ; CHECK-NEXT:    jmp .LBB0_4
 ; CHECK-NEXT:  .LBB0_3: # %branchB
 ; CHECK-NEXT:    movq externalB at GOTPCREL(%rip), %rax
@@ -56,14 +56,14 @@ define void @test() personality ptr @__gxx_personality_v0  {
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    movq externalD at GOTPCREL(%rip), %rax
 ; CHECK-NEXT:    movl (%rax), %eax
-; CHECK-NEXT:    movl %eax, (%rsp) # 4-byte Spill
+; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:  .Ltmp0: # EH_LABEL
 ; CHECK-NEXT:    callq maythrow at PLT
-; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:  .Ltmp1: # EH_LABEL
 ; CHECK-NEXT:  .LBB0_4: # %end
-; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 56
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
@@ -79,13 +79,13 @@ define void @test() personality ptr @__gxx_personality_v0  {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB0_2: # %lpad
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:  .Ltmp4:
-; CHECK-NEXT:    movq %rax, %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:  .Ltmp4: # EH_LABEL
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT:    addl (%rsp), %edi # 4-byte Folded Reload
+; CHECK-NEXT:    addl {{[-0-9]+}}(%r{{[sb]}}p), %edi # 4-byte Folded Reload
 ; CHECK-NEXT:    callq cleanup at PLT
-; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; CHECK-NEXT:    callq _Unwind_Resume at PLT
   %1 = load i1, ptr @external_bool
   br i1 %1, label %branchA, label %branchB
diff --git a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
index 0d8dc274da1f2..5f1b34897a3ef 100644
--- a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -94,23 +94,23 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne LBB0_11
 ; CHECK-NEXT:  ## %bb.12: ## %while.body200.preheader
-; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    leaq LJTI0_0(%rip), %rdx
-; CHECK-NEXT:    leaq LJTI0_1(%rip), %r14
+; CHECK-NEXT:    leaq LJTI0_1(%rip), %r13
 ; CHECK-NEXT:    movb $1, %sil
 ; CHECK-NEXT:    movl $0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
-; CHECK-NEXT:    xorl %r12d, %r12d
+; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    jmp LBB0_13
 ; CHECK-NEXT:  LBB0_43: ## %while.cond1037.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %r13b, %r13b
+; CHECK-NEXT:    testb %r15b, %r15b
 ; CHECK-NEXT:    je LBB0_54
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_20: ## %while.cond197.backedge
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    decl %ebp
 ; CHECK-NEXT:    testl %ebp, %ebp
-; CHECK-NEXT:    movl %ebx, %r12d
+; CHECK-NEXT:    movl %ebx, %r14d
 ; CHECK-NEXT:    jle LBB0_21
 ; CHECK-NEXT:  LBB0_13: ## %while.body200
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
@@ -121,18 +121,18 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    ja LBB0_14
 ; CHECK-NEXT:  ## %bb.55: ## %while.body200
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    movslq (%r14,%rax,4), %rax
-; CHECK-NEXT:    addq %r14, %rax
+; CHECK-NEXT:    movslq (%r13,%rax,4), %rax
+; CHECK-NEXT:    addq %r13, %rax
 ; CHECK-NEXT:    jmpq *%rax
 ; CHECK-NEXT:  LBB0_25: ## %sw.bb474
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %r13b, %r13b
-; CHECK-NEXT:    ## implicit-def: $r15
+; CHECK-NEXT:    testb %r15b, %r15b
+; CHECK-NEXT:    ## implicit-def: $r12
 ; CHECK-NEXT:    jne LBB0_33
 ; CHECK-NEXT:  ## %bb.26: ## %do.body479.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    testb %r13b, %r13b
-; CHECK-NEXT:    ## implicit-def: $r15
+; CHECK-NEXT:    testb %r15b, %r15b
+; CHECK-NEXT:    ## implicit-def: $r12
 ; CHECK-NEXT:    jne LBB0_33
 ; CHECK-NEXT:  ## %bb.27: ## %land.rhs485.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
@@ -141,8 +141,8 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_31: ## %do.body479.backedge
 ; CHECK-NEXT:    ## in Loop: Header=BB0_28 Depth=2
-; CHECK-NEXT:    leaq 1(%r15), %rax
-; CHECK-NEXT:    testb %r13b, %r13b
+; CHECK-NEXT:    leaq 1(%r12), %rax
+; CHECK-NEXT:    testb %r15b, %r15b
 ; CHECK-NEXT:    je LBB0_32
 ; CHECK-NEXT:  LBB0_28: ## %land.rhs485
 ; CHECK-NEXT:    ## Parent Loop BB0_13 Depth=1
@@ -151,7 +151,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    jne LBB0_54
 ; CHECK-NEXT:  ## %bb.29: ## %cond.true.i.i2780
 ; CHECK-NEXT:    ## in Loop: Header=BB0_28 Depth=2
-; CHECK-NEXT:    movq %rax, %r15
+; CHECK-NEXT:    movq %rax, %r12
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne LBB0_31
 ; CHECK-NEXT:  ## %bb.30: ## %lor.rhs500
@@ -197,10 +197,10 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    jmp LBB0_20
 ; CHECK-NEXT:  LBB0_32: ## %if.end517.loopexitsplit
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    incq %r15
+; CHECK-NEXT:    incq %r12
 ; CHECK-NEXT:  LBB0_33: ## %if.end517
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    leal -324(%r12), %eax
+; CHECK-NEXT:    leal -324(%r14), %eax
 ; CHECK-NEXT:    cmpl $59, %eax
 ; CHECK-NEXT:    ja LBB0_34
 ; CHECK-NEXT:  ## %bb.56: ## %if.end517
@@ -210,11 +210,11 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    jb LBB0_37
 ; CHECK-NEXT:  LBB0_34: ## %if.end517
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    cmpl $11, %r12d
+; CHECK-NEXT:    cmpl $11, %r14d
 ; CHECK-NEXT:    je LBB0_37
 ; CHECK-NEXT:  ## %bb.35: ## %if.end517
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
-; CHECK-NEXT:    cmpl $24, %r12d
+; CHECK-NEXT:    cmpl $24, %r14d
 ; CHECK-NEXT:    je LBB0_37
 ; CHECK-NEXT:  ## %bb.36: ## %if.then532
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
@@ -224,12 +224,12 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:  LBB0_37: ## %for.cond534
 ; CHECK-NEXT:    ## Parent Loop BB0_13 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    testb %r13b, %r13b
+; CHECK-NEXT:    testb %r15b, %r15b
 ; CHECK-NEXT:    jne LBB0_37
 ; CHECK-NEXT:  ## %bb.38: ## %for.cond542.preheader
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    movb $0, (%r15)
+; CHECK-NEXT:    movb $0, (%r12)
 ; CHECK-NEXT:    leaq LJTI0_0(%rip), %rdx
 ; CHECK-NEXT:    jmp LBB0_20
 ; CHECK-NEXT:    .p2align 4
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening.ll b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
index 0fd81cdad4d2b..2773221921944 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
@@ -32,11 +32,8 @@ entry:
 define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2, ptr %ptr3) speculative_load_hardening {
 ; X64-LABEL: test_basic_conditions:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rbx
-; X64-NEXT:    .cfi_def_cfa_offset 16
-; X64-NEXT:    subq $16, %rsp
+; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    .cfi_def_cfa_offset 32
-; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movq %rsp, %rax
 ; X64-NEXT:    movq $-1, %r10
 ; X64-NEXT:    sarq $63, %rax
@@ -51,9 +48,7 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
 ; X64-NEXT:  .LBB1_8: # %exit
 ; X64-NEXT:    shlq $47, %rax
 ; X64-NEXT:    orq %rax, %rsp
-; X64-NEXT:    addq $16, %rsp
-; X64-NEXT:    .cfi_def_cfa_offset 16
-; X64-NEXT:    popq %rbx
+; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB1_4: # %then2
@@ -76,16 +71,18 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
 ; X64-NEXT:    orq %rax, %rdi
 ; X64-NEXT:    movl (%r8,%rdi,4), %esi
 ; X64-NEXT:    orl %eax, %esi
-; X64-NEXT:    movq (%r9), %rbx
-; X64-NEXT:    orq %rax, %rbx
-; X64-NEXT:    addl (%rbx), %esi
+; X64-NEXT:    movq (%r9), %rcx
+; X64-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-NEXT:    orq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; X64-NEXT:    addl (%rcx), %esi
 ; X64-NEXT:    shlq $47, %rax
 ; X64-NEXT:    # kill: def $edi killed $edi killed $rdi
 ; X64-NEXT:    orq %rax, %rsp
 ; X64-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-NEXT:    callq leak at PLT
 ; X64-NEXT:  .Lslh_ret_addr0:
-; X64-NEXT:    movq %rbx, %rcx
+; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
 ; X64-NEXT:    movq %rsp, %rax
 ; X64-NEXT:    movq -{{[0-9]+}}(%rsp), %rdx
@@ -101,11 +98,8 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
 ;
 ; X64-LFENCE-LABEL: test_basic_conditions:
 ; X64-LFENCE:       # %bb.0: # %entry
-; X64-LFENCE-NEXT:    pushq %rbx
-; X64-LFENCE-NEXT:    .cfi_def_cfa_offset 16
-; X64-LFENCE-NEXT:    subq $16, %rsp
+; X64-LFENCE-NEXT:    subq $24, %rsp
 ; X64-LFENCE-NEXT:    .cfi_def_cfa_offset 32
-; X64-LFENCE-NEXT:    .cfi_offset %rbx, -16
 ; X64-LFENCE-NEXT:    testl %edi, %edi
 ; X64-LFENCE-NEXT:    jne .LBB1_6
 ; X64-LFENCE-NEXT:  # %bb.1: # %then1
@@ -128,21 +122,21 @@ define void @test_basic_conditions(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2,
 ; X64-LFENCE-NEXT:    addl (%r8), %eax
 ; X64-LFENCE-NEXT:    movslq %eax, %rdi
 ; X64-LFENCE-NEXT:    movl (%r8,%rdi,4), %esi
-; X64-LFENCE-NEXT:    movq (%r9), %rbx
-; X64-LFENCE-NEXT:    addl (%rbx), %esi
+; X64-LFENCE-NEXT:    movq (%r9), %rax
+; X64-LFENCE-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; X64-LFENCE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; X64-LFENCE-NEXT:    addl (%rax), %esi
 ; X64-LFENCE-NEXT:    # kill: def $edi killed $edi killed $rdi
 ; X64-LFENCE-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; X64-LFENCE-NEXT:    callq leak at PLT
-; X64-LFENCE-NEXT:    movq %rbx, %rax
+; X64-LFENCE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; X64-LFENCE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
 ; X64-LFENCE-NEXT:  .LBB1_5: # %merge
 ; X64-LFENCE-NEXT:    movslq (%rax), %rax
 ; X64-LFENCE-NEXT:    movl $0, (%r8,%rax,4)
 ; X64-LFENCE-NEXT:  .LBB1_6: # %exit
 ; X64-LFENCE-NEXT:    lfence
-; X64-LFENCE-NEXT:    addq $16, %rsp
-; X64-LFENCE-NEXT:    .cfi_def_cfa_offset 16
-; X64-LFENCE-NEXT:    popq %rbx
+; X64-LFENCE-NEXT:    addq $24, %rsp
 ; X64-LFENCE-NEXT:    .cfi_def_cfa_offset 8
 ; X64-LFENCE-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/throws-cfi-fp.ll b/llvm/test/CodeGen/X86/throws-cfi-fp.ll
index c6fc8bda52010..cf5b9e0daad6f 100644
--- a/llvm/test/CodeGen/X86/throws-cfi-fp.ll
+++ b/llvm/test/CodeGen/X86/throws-cfi-fp.ll
@@ -26,12 +26,9 @@ define void @_Z6throwsv() #0 personality ptr @__gxx_personality_v0 {
 ; PEI-NEXT:    frame-setup CFI_INSTRUCTION offset $rbp, -16
 ; PEI-NEXT:    $rbp = frame-setup MOV64rr $rsp
 ; PEI-NEXT:    frame-setup CFI_INSTRUCTION def_cfa_register $rbp
-; PEI-NEXT:    frame-setup PUSH64r killed $rbx, implicit-def $rsp, implicit $rsp
-; PEI-NEXT:    frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp
-; PEI-NEXT:    {{^ +}}CFI_INSTRUCTION offset $rbx, -24
+; PEI-NEXT:    $rsp = frame-setup SUB64ri32 $rsp, 16, implicit-def dead $eflags
 ; PEI:         bb.4.try.cont:
-; PEI-NEXT:      $rsp = frame-destroy ADD64ri32 $rsp, 8, implicit-def dead $eflags
-; PEI-NEXT:      $rbx = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+; PEI-NEXT:      $rsp = frame-destroy ADD64ri32 $rsp, 16, implicit-def dead $eflags
 ; PEI-NEXT:      $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp
 ; PEI-NEXT:      frame-destroy CFI_INSTRUCTION def_cfa $rsp, 8
 ; PEI-NEXT:      RET 0
diff --git a/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll b/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll
index 6663d91b75355..795c58f5d7d40 100644
--- a/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll
+++ b/llvm/test/CodeGen/X86/throws-cfi-no-fp.ll
@@ -13,7 +13,7 @@ $__clang_call_terminate = comdat any
 define void @_Z6throwsv() personality ptr @__gxx_personality_v0 {
 
 ; CHECK-LABEL:   _Z6throwsv:
-; CHECK:         popq	%rbx
+; CHECK:         popq	%rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    .LBB0_1:



More information about the llvm-branch-commits mailing list