[llvm-branch-commits] [llvm] [AMDGPU] Use banked VGPR pressure in mayCauseSpilling (PR #243965)

Dhruva Chakrabarti via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 18 10:52:40 PST 2026


https://github.com/dhruvachak created https://github.com/llvm/llvm-project/pull/243965

With AVGPR banking, the excess checks in checkScheduling are bank-aware but the before/after comparison in mayCauseSpilling was not. Make it measure VGPR excess from banked per-point peaks, before and after scheduling, and compare the combined-file excess first, since excess in one bank can be absorbed by the other.

Depends on https://github.com/llvm/llvm-project/pull/243962

>From c0f3897c3371a7630bdc8de5f101e0109556ae11 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 17 Nov 2026 21:49:28 -0600
Subject: [PATCH] [AMDGPU] Use banked VGPR pressure in mayCauseSpilling

With AVGPR banking, the excess checks in checkScheduling are bank-aware but
the before/after comparison in mayCauseSpilling was not. Make it measure
VGPR excess from banked per-point peaks, before and after scheduling, and
compare the combined-file excess first, since excess in one bank can be
absorbed by the other.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 llvm/lib/Target/AMDGPU/GCNRegPressure.cpp     |  31 +++++-
 llvm/lib/Target/AMDGPU/GCNRegPressure.h       |  36 ++++++-
 llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp   |  58 ++++++++--
 llvm/lib/Target/AMDGPU/GCNSchedStrategy.h     |  12 +++
 .../AMDGPU/av-split-dead-valno-crash.ll       | 100 +++++++++---------
 ...ite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll |  52 ++++-----
 6 files changed, 203 insertions(+), 86 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 050903724be890..dbb9d7bc8aacca 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -142,11 +142,29 @@ struct RegExcess {
     // AGPR excess pressure conditions
     AGPR = std::max(static_cast<int>(RP.getAGPRNum() - ArchVGPRLimit), 0);
   }
+
+  /// Measures the vector register excess from the banked \p Peaks instead.
+  RegExcess(const MachineFunction &MF, const GCNRegPressure &RP,
+            const GCNRegPressure::BankedVGPRPeaks &Peaks,
+            const GCNRegPressure::BankedComparison &Limits)
+      : RegExcess(MF, RP) {
+    unsigned VGPRForSGPRSpills =
+        divideCeil(SGPR, MF.getSubtarget<GCNSubtarget>().getWavefrontSize());
+    VGPR =
+        std::max(static_cast<int>(Peaks.getUnifiedVGPRNum() +
+                                  VGPRForSGPRSpills - Limits.UnifiedVGPRLimit),
+                 0);
+    ArchVGPR = std::max(static_cast<int>(Peaks.ArchVGPRs + VGPRForSGPRSpills -
+                                         Limits.ArchVGPRLimit),
+                        0);
+    AGPR = std::max(static_cast<int>(Peaks.AGPRs - Limits.AGPRLimit), 0);
+  }
 };
 } // namespace
 
 bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
-                          unsigned MaxOccupancy) const {
+                          unsigned MaxOccupancy,
+                          const BankedComparison *Banked) const {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   unsigned DynamicVGPRBlockSize =
       MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
@@ -172,8 +190,10 @@ bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
 
   unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
 
-  RegExcess Excess(MF, *this);
-  RegExcess OtherExcess(MF, O);
+  RegExcess Excess = Banked ? RegExcess(MF, *this, Banked->Peaks, *Banked)
+                            : RegExcess(MF, *this);
+  RegExcess OtherExcess =
+      Banked ? RegExcess(MF, O, Banked->OtherPeaks, *Banked) : RegExcess(MF, O);
 
   unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
 
@@ -183,6 +203,11 @@ bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,
   // Give second precedence to the reduced number of spills to hold the register
   // pressure.
   if (ExcessRP || OtherExcessRP) {
+    // The allocator can move excess in one bank into free registers of the
+    // other, so only excess in the combined file is sure to spill.
+    if (Banked && Excess.VGPR != OtherExcess.VGPR)
+      return Excess.VGPR < OtherExcess.VGPR;
+
     // The difference in excess VGPR pressure, after including VGPRs used for
     // SGPR spills
     int VGPRDiff =
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 1e9ce8dbf5d7cd..8ba9af94fd6500 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -115,6 +115,35 @@ struct GCNRegPressure {
     return {Arch + ToArch + Flex, Acc + ToAcc};
   }
 
+  /// Peak ArchVGPR32 and AccVGPR32 pressure over a set of program points, after
+  /// distributing the AVGPR pressure of each point with getBankedVGPRNum().
+  struct BankedVGPRPeaks {
+    unsigned ArchVGPRs = 0;
+    unsigned AGPRs = 0;
+
+    void add(const GCNRegPressure &RP, unsigned ArchLimit, unsigned AccLimit) {
+      auto [Arch, Acc] = RP.getBankedVGPRNum(ArchLimit, AccLimit);
+      ArchVGPRs = std::max(ArchVGPRs, Arch);
+      AGPRs = std::max(AGPRs, Acc);
+    }
+
+    /// \returns the combined-file pressure. The banks use separate ranges of
+    /// the file, so this is the sum of both peaks.
+    unsigned getUnifiedVGPRNum() const {
+      return getBankedUnifiedVGPRNum(ArchVGPRs, AGPRs);
+    }
+  };
+
+  /// The banked peaks of two pressures compared by less(), and the limits their
+  /// vector register excess is measured against.
+  struct BankedComparison {
+    BankedVGPRPeaks Peaks;
+    BankedVGPRPeaks OtherPeaks;
+    unsigned ArchVGPRLimit;
+    unsigned AGPRLimit;
+    unsigned UnifiedVGPRLimit;
+  };
+
   unsigned getVGPRTuplesWeight() const {
     return std::max(Value[TOTAL_KINDS + VGPR] + Value[TOTAL_KINDS + AVGPR],
                     Value[TOTAL_KINDS + AGPR]);
@@ -168,8 +197,13 @@ struct GCNRegPressure {
   /// determine that SGPR pressure is not important)
   /// 4. Less raw register pressure (first preference to VGPR tuples if we
   /// determine that SGPR pressure is not important)
+  ///
+  /// If \p Banked is set, the vector register excess is measured from the
+  /// banked peaks in \p Banked, and the combined-file excess is compared before
+  /// the per-bank excess.
   bool less(const MachineFunction &MF, const GCNRegPressure &O,
-            unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
+            unsigned MaxOccupancy = std::numeric_limits<unsigned>::max(),
+            const BankedComparison *Banked = nullptr) const;
 
   bool operator==(const GCNRegPressure &O) const { return Value == O.Value; }
 
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 7302f18c77aa2e..2bf51596644d60 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1638,6 +1638,21 @@ bool GCNSchedStage::initGCNRegion() {
 
   PressureBefore = DAG.Pressure[RegionIdx];
 
+  BankedPeaksBefore = {};
+  if (useBankedSpillCheck()) {
+    unsigned ArchLimit, AccLimit;
+    std::tie(ArchLimit, AccLimit) = getSpillCheckBankLimits();
+    // If no point has AVGPRs beyond the ArchVGPR bank, banking the region
+    // maximum is exact. Otherwise bank each point.
+    if (PressureBefore.getArchVGPRNum() <= ArchLimit) {
+      BankedPeaksBefore.add(PressureBefore, ArchLimit, AccLimit);
+    } else {
+      DAG.getRealRegPressure(RegionIdx, [&](const GCNRegPressure &RP) {
+        BankedPeaksBefore.add(RP, ArchLimit, AccLimit);
+      });
+    }
+  }
+
   LLVM_DEBUG(
       dbgs() << "Pressure before scheduling:\nRegion live-ins:"
              << print(DAG.LiveIns[RegionIdx], DAG.MRI)
@@ -1766,6 +1781,10 @@ void GCNSchedStage::checkScheduling() {
   unsigned ArchVGPRBankLimit, AGPRBankLimit;
   std::tie(ArchVGPRBankLimit, AGPRBankLimit) = S.getBankLimits();
   unsigned BankedArchVGPRPeak = 0, BankedAGPRPeak = 0, BankedUnifiedPeak = 0;
+  BankedPeaksAfter = {};
+  bool BankedSpillCheck = useBankedSpillCheck();
+  unsigned SpillCheckArchLimit, SpillCheckAccLimit;
+  std::tie(SpillCheckArchLimit, SpillCheckAccLimit) = getSpillCheckBankLimits();
   std::function<void(const GCNRegPressure &)> TrackBankedPeaks;
   if (AVGPRBanking) {
     TrackBankedPeaks = [&](const GCNRegPressure &RP) {
@@ -1776,6 +1795,8 @@ void GCNSchedStage::checkScheduling() {
       BankedUnifiedPeak =
           std::max(BankedUnifiedPeak,
                    GCNRegPressure::getBankedUnifiedVGPRNum(ArchNum, AccNum));
+      if (BankedSpillCheck)
+        BankedPeaksAfter.add(RP, SpillCheckArchLimit, SpillCheckAccLimit);
     };
   }
 
@@ -1792,6 +1813,10 @@ void GCNSchedStage::checkScheduling() {
       UnifiedVGPRFile ? S.UnifiedVGPRCriticalLimit : S.VGPRCriticalLimit;
   // Fitting the combined occupancy budget does not rule out spilling in one
   // bank. Keep the per-bank excess checks before taking this early exit.
+  // These checks are not banked: when they pass, all AVGPRs fit in the
+  // ArchVGPR bank, so banking would not change any count. Banking them would
+  // only take the exit, skipping the revert checks below, for regions whose
+  // AVGPRs need the AGPR bank.
   if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
       PressureAfter.getVGPRNum(UnifiedVGPRFile) <= VectorCriticalLimit &&
       (!UnifiedVGPRFile ||
@@ -2151,14 +2176,35 @@ void LiveIntervalRPStage::finalizeGCNRegion() {
   GCNSchedStage::finalizeGCNRegion();
 }
 
+bool GCNSchedStage::useBankedSpillCheck() const {
+  return ST.getCLOpts().avgpr_banking && ST.hasGFX90AInsts();
+}
+
+std::pair<unsigned, unsigned> GCNSchedStage::getSpillCheckBankLimits() const {
+  auto [ArchLimit, AccLimit] = S.getBankLimits();
+  return {ArchLimit - std::min(S.ErrorMargin, ArchLimit),
+          AccLimit - std::min(S.ErrorMargin, AccLimit)};
+}
+
 bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) {
-  if (WavesAfter <= MFI.getMinWavesPerEU() && isRegionWithExcessRP() &&
-      !PressureAfter.less(MF, PressureBefore)) {
-    LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n");
-    return true;
-  }
+  if (WavesAfter > MFI.getMinWavesPerEU() || !isRegionWithExcessRP())
+    return false;
 
-  return false;
+  std::optional<GCNRegPressure::BankedComparison> Banked;
+  if (useBankedSpillCheck()) {
+    auto [ArchLimit, AccLimit] = getSpillCheckBankLimits();
+    unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
+    Banked = GCNRegPressure::BankedComparison{
+        BankedPeaksAfter, BankedPeaksBefore, ArchLimit, AccLimit,
+        MaxVGPRs - std::min(S.ErrorMargin, MaxVGPRs)};
+  }
+  if (PressureAfter.less(MF, PressureBefore,
+                         std::numeric_limits<unsigned>::max(),
+                         Banked ? &*Banked : nullptr))
+    return false;
+
+  LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n");
+  return true;
 }
 
 void GCNSchedStage::modifyRegionSchedule(unsigned RegionIdx,
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 9ae631b5e87c24..0972168e07a507 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -416,8 +416,20 @@ class GCNSchedStage {
   // RP after scheduling the current region.
   GCNRegPressure PressureAfter;
 
+  // Banked VGPR peaks before and after scheduling the current region, for
+  // mayCauseSpilling(). Only set if useBankedSpillCheck().
+  GCNRegPressure::BankedVGPRPeaks BankedPeaksBefore;
+  GCNRegPressure::BankedVGPRPeaks BankedPeaksAfter;
+
   std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
 
+  /// \returns true if mayCauseSpilling() compares banked VGPR pressure.
+  bool useBankedSpillCheck() const;
+
+  /// \returns the ArchVGPR and AGPR limits that mayCauseSpilling() distributes
+  /// AVGPR pressure against.
+  std::pair<unsigned, unsigned> getSpillCheckBankLimits() const;
+
   GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG);
 
 public:
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 7d134b10b33bef..6948dd01354843 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -4,81 +4,81 @@
 define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2, i1 %cond.i.i.i2295, ptr addrspace(1) %ptr, ptr %ptr1) #0 {
 ; CHECK-LABEL: vgpr_mfma_pass_av_split_crash:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
 ; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x8
-; CHECK-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x10
-; CHECK-NEXT:    s_mov_b64 s[14:15], 0
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x9037ab78
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x3e21eeb6
+; CHECK-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x0
+; CHECK-NEXT:    s_mov_b64 s[12:13], 0
+; CHECK-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x10
+; CHECK-NEXT:    v_mov_b32_e32 v30, 0x9037ab78
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s0, 0
-; CHECK-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT:    s_cselect_b64 s[10:11], -1, 0
 ; CHECK-NEXT:    s_bitcmp1_b32 s0, 8
-; CHECK-NEXT:    s_cselect_b64 s[12:13], -1, 0
-; CHECK-NEXT:    v_mov_b64_e32 v[30:31], s[10:11]
-; CHECK-NEXT:    v_mov_b32_e32 v4, 0xa17f65f6
-; CHECK-NEXT:    v_mov_b32_e32 v5, 0xbe927e4f
-; CHECK-NEXT:    v_mov_b32_e32 v6, 0x19f4ec90
-; CHECK-NEXT:    v_mov_b32_e32 v7, 0x3efa01a0
-; CHECK-NEXT:    v_mov_b32_e32 v8, 0x16c16967
-; CHECK-NEXT:    v_mov_b32_e32 v9, 0xbf56c16c
-; CHECK-NEXT:    v_mov_b32_e32 v10, 0x69efb384
-; CHECK-NEXT:    v_mov_b32_e32 v11, 0x3f4b2bb0
+; CHECK-NEXT:    s_cselect_b64 s[14:15], -1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v31, 0x3e21eeb6
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xa17f65f6
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0xbe927e4f
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x19f4ec90
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0x3efa01a0
+; CHECK-NEXT:    v_mov_b32_e32 v6, 0x16c16967
+; CHECK-NEXT:    v_mov_b32_e32 v7, 0xbf56c16c
+; CHECK-NEXT:    v_mov_b32_e32 v8, 0x69efb384
+; CHECK-NEXT:    v_mov_b32_e32 v9, 0x3f4b2bb0
+; CHECK-NEXT:    v_mov_b32_e32 v10, 0xa57d9582
+; CHECK-NEXT:    v_mov_b32_e32 v11, 0xbf8c6ea4
+; CHECK-NEXT:    v_mov_b32_e32 v12, 0xe82d3ff0
+; CHECK-NEXT:    v_mov_b32_e32 v13, 0xbfa59976
+; CHECK-NEXT:    v_mov_b32_e32 v14, 0x8427b883
+; CHECK-NEXT:    v_mov_b32_e32 v15, 0x3fae1bb4
+; CHECK-NEXT:    v_mov_b32_e32 v16, 0x57b87036
+; CHECK-NEXT:    v_mov_b32_e32 v17, 0x3fb3b136
+; CHECK-NEXT:    v_mov_b32_e32 v18, 0x55555523
+; CHECK-NEXT:    v_mov_b32_e32 v19, 0xbfd55555
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b64_e32 v[12:13], 0
-; CHECK-NEXT:    v_mov_b32_e32 v14, 0xa57d9582
-; CHECK-NEXT:    v_mov_b32_e32 v15, 0xbf8c6ea4
-; CHECK-NEXT:    v_mov_b32_e32 v16, 0xe82d3ff0
-; CHECK-NEXT:    v_mov_b32_e32 v17, 0xbfa59976
-; CHECK-NEXT:    v_mov_b32_e32 v18, 0x8427b883
-; CHECK-NEXT:    v_mov_b32_e32 v19, 0x3fae1bb4
-; CHECK-NEXT:    v_mov_b32_e32 v20, 0x57b87036
-; CHECK-NEXT:    v_mov_b32_e32 v21, 0x3fb3b136
-; CHECK-NEXT:    v_mov_b32_e32 v22, 0x55555523
-; CHECK-NEXT:    v_mov_b32_e32 v23, 0xbfd55555
+; CHECK-NEXT:    v_mov_b64_e32 v[20:21], s[6:7]
+; CHECK-NEXT:    v_mov_b64_e32 v[22:23], 0
 ; CHECK-NEXT:    ; implicit-def: $agpr0_agpr1
-; CHECK-NEXT:    ; implicit-def: $sgpr10_sgpr11
+; CHECK-NEXT:    ; implicit-def: $sgpr16_sgpr17
 ; CHECK-NEXT:    s_branch .LBB0_3
 ; CHECK-NEXT:  .LBB0_1: ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    v_mov_b64_e32 v[24:25], s[6:7]
+; CHECK-NEXT:    v_mov_b64_e32 v[24:25], s[8:9]
 ; CHECK-NEXT:    v_accvgpr_write_b32 a0, v24
 ; CHECK-NEXT:    v_accvgpr_write_b32 a1, v25
-; CHECK-NEXT:    s_mov_b64 s[16:17], s[10:11]
-; CHECK-NEXT:    s_mov_b64 s[14:15], 0
+; CHECK-NEXT:    s_mov_b64 s[6:7], s[16:17]
+; CHECK-NEXT:    s_mov_b64 s[12:13], 0
 ; CHECK-NEXT:  .LBB0_2: ; %._crit_edge2105.i.i.i2330
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    global_store_dwordx2 v0, v[12:13], s[8:9]
-; CHECK-NEXT:    s_mov_b64 s[10:11], s[16:17]
+; CHECK-NEXT:    global_store_dwordx2 v0, v[22:23], s[4:5]
+; CHECK-NEXT:    s_mov_b64 s[16:17], s[6:7]
 ; CHECK-NEXT:  .LBB0_3: ; %._crit_edge1942.i.i.i3548
 ; CHECK-NEXT:    ; =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ; Child Loop BB0_6 Depth 2
-; CHECK-NEXT:    s_and_b64 s[0:1], s[12:13], exec
+; CHECK-NEXT:    s_and_b64 s[0:1], s[14:15], exec
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB0_1
 ; CHECK-NEXT:  ; %bb.4: ; %.preheader1868.i.i.i3244
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    s_and_b64 s[2:3], s[4:5], exec
-; CHECK-NEXT:    s_mov_b64 s[16:17], 0
+; CHECK-NEXT:    s_and_b64 s[2:3], s[10:11], exec
+; CHECK-NEXT:    s_mov_b64 s[6:7], 0
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB0_2
 ; CHECK-NEXT:  ; %bb.5: ; %.preheader1855.i.i.i3329.preheader
 ; CHECK-NEXT:    ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT:    flat_load_dwordx2 v[24:25], v[30:31]
-; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[14:15]
+; CHECK-NEXT:    flat_load_dwordx2 v[24:25], v[20:21]
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[10:11]
 ; CHECK-NEXT:    v_accvgpr_write_b32 a2, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a3, 0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[24:25], v[2:3]
+; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[24:25], v[30:31]
+; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[26:27], v[2:3]
 ; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[26:27], v[4:5]
 ; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[26:27], v[6:7]
 ; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[26:27], v[8:9]
-; CHECK-NEXT:    v_fma_f64 v[26:27], 0, v[26:27], v[10:11]
 ; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[16:17]
+; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[12:13]
 ; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[18:19]
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[14:15]
 ; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[20:21]
+; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[16:17]
 ; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[22:23]
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[18:19]
 ; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
 ; CHECK-NEXT:  .LBB0_6: ; %.preheader1855.i.i.i3329
 ; CHECK-NEXT:    ; Parent Loop BB0_3 Depth=1
@@ -108,13 +108,13 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
 ; CHECK-NEXT:    v_accvgpr_write_b32 a0, v24
 ; CHECK-NEXT:    v_accvgpr_write_b32 a1, v25
 ; CHECK-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s[0:1]
-; CHECK-NEXT:    s_and_b64 s[0:1], s[4:5], exec
+; CHECK-NEXT:    s_and_b64 s[0:1], s[10:11], exec
 ; CHECK-NEXT:    v_mov_b32_e32 v27, v26
-; CHECK-NEXT:    s_cselect_b32 s17, 0, s11
-; CHECK-NEXT:    s_cselect_b32 s16, 0, s10
-; CHECK-NEXT:    s_cselect_b32 s15, s15, 0
-; CHECK-NEXT:    s_cselect_b32 s14, s14, 0
-; CHECK-NEXT:    global_store_dwordx2 v0, v[26:27], s[8:9]
+; CHECK-NEXT:    s_cselect_b32 s7, 0, s17
+; CHECK-NEXT:    s_cselect_b32 s6, 0, s16
+; CHECK-NEXT:    s_cselect_b32 s13, s13, 0
+; CHECK-NEXT:    s_cselect_b32 s12, s12, 0
+; CHECK-NEXT:    global_store_dwordx2 v0, v[26:27], s[4:5]
 ; CHECK-NEXT:    s_branch .LBB0_2
 ; CHECK-NEXT:  .LBB0_12: ; %DummyReturnBlock
 ; CHECK-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
index 3bd5a6e98c80ea..3aab0280b1f8bb 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
@@ -19,7 +19,7 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_load_dword s8, s[4:5], 0x10
 ; CHECK-NEXT:    s_load_dwordx4 s[16:19], s[4:5], 0x20
 ; CHECK-NEXT:    s_load_dword s9, s[4:5], 0x30
-; CHECK-NEXT:    s_mov_b64 s[54:55], -1
+; CHECK-NEXT:    s_load_dword s28, s[4:5], 0x90
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s21, 0
 ; CHECK-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -36,24 +36,22 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; CHECK-NEXT:    s_bitcmp1_b32 s9, 0
 ; CHECK-NEXT:    s_cselect_b64 s[38:39], -1, 0
-; CHECK-NEXT:    s_mov_b64 s[52:53], 0
+; CHECK-NEXT:    s_mov_b64 s[54:55], -1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s8, 0
 ; CHECK-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x60
-; CHECK-NEXT:    s_load_dword s28, s[4:5], 0x90
 ; CHECK-NEXT:    s_cselect_b64 s[40:41], -1, 0
-; CHECK-NEXT:    v_mov_b32_e32 v9, 0
-; CHECK-NEXT:    v_mov_b32_e32 v8, 0
-; CHECK-NEXT:    v_mov_b32_e32 v7, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s28, 0
 ; CHECK-NEXT:    s_mov_b32 s28, 0
 ; CHECK-NEXT:    s_cselect_b64 s[42:43], -1, 0
-; CHECK-NEXT:    s_lshl_b32 s21, s21, 1
+; CHECK-NEXT:    s_lshl_b32 s44, s21, 1
 ; CHECK-NEXT:    s_mov_b32 s29, s28
 ; CHECK-NEXT:    s_mov_b32 s30, s28
 ; CHECK-NEXT:    s_mov_b32 s31, s28
-; CHECK-NEXT:    v_mov_b32_e32 v6, 0
+; CHECK-NEXT:    s_mov_b64 s[52:53], 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a0, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a1, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a2, 0
@@ -90,21 +88,21 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    v_accvgpr_write_b32 a17, s29
 ; CHECK-NEXT:    v_accvgpr_write_b32 a18, s30
 ; CHECK-NEXT:    v_accvgpr_write_b32 a19, s31
-; CHECK-NEXT:    v_accvgpr_write_b32 a24, s21
-; CHECK-NEXT:    v_accvgpr_write_b32 a25, s22
 ; CHECK-NEXT:    s_mov_b64 s[60:61], 0
 ; CHECK-NEXT:    s_mov_b64 s[58:59], 0
 ; CHECK-NEXT:    s_mov_b64 s[56:57], 0
 ; CHECK-NEXT:    s_mov_b64 s[50:51], 0
-; CHECK-NEXT:    s_mov_b32 s64, s28
+; CHECK-NEXT:    s_mov_b32 s63, s28
 ; CHECK-NEXT:    s_mov_b64 s[48:49], 0
 ; CHECK-NEXT:    s_mov_b64 s[46:47], 0
 ; CHECK-NEXT:    s_mov_b32 s21, s28
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
 ; CHECK-NEXT:    s_mov_b32 s33, s28
 ; CHECK-NEXT:    s_mov_b32 s62, s28
-; CHECK-NEXT:    s_mov_b32 s63, s28
+; CHECK-NEXT:    s_mov_b32 s64, s28
 ; CHECK-NEXT:    s_mov_b32 s65, s28
-; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    v_accvgpr_write_b32 a24, s44
+; CHECK-NEXT:    v_accvgpr_write_b32 a25, s22
 ; CHECK-NEXT:  .LBB0_1: ; %bbl97
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[52:53], exec
@@ -114,15 +112,17 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    v_mov_b32_e32 v14, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v15, s52
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[60:61], exec
-; CHECK-NEXT:    v_accvgpr_read_b32 v0, a24
+; CHECK-NEXT:    v_accvgpr_read_b32 v4, a24
 ; CHECK-NEXT:    s_cselect_b32 s52, 0x3f803f80, 0
-; CHECK-NEXT:    ds_write_b128 v0, v[12:15]
+; CHECK-NEXT:    ds_write_b128 v4, v[12:15]
 ; CHECK-NEXT:    v_mov_b32_e32 v12, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v13, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v14, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v15, s52
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[58:59], exec
 ; CHECK-NEXT:    s_mov_b64 s[44:45], s[54:55]
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_cselect_b32 s52, s11, 0
 ; CHECK-NEXT:    s_cselect_b32 s53, s10, 0
 ; CHECK-NEXT:    s_cselect_b32 s54, s9, 0
@@ -134,15 +134,15 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    v_mov_b32_e32 v15, s52
 ; CHECK-NEXT:    v_accvgpr_read_b32 v5, a25
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, v9, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, v8, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, v7, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, v6, s[0:1]
-; CHECK-NEXT:    s_or_b32 s52, s64, s20
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[0:1]
+; CHECK-NEXT:    s_or_b32 s52, s63, s20
 ; CHECK-NEXT:    ds_write_b128 v5, v[12:15]
 ; CHECK-NEXT:    ds_write_b128 v4, v[0:3]
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s52
-; CHECK-NEXT:    buffer_load_dwordx4 v[6:9], v0, s[28:31], 0 offen
+; CHECK-NEXT:    buffer_load_dwordx4 v[0:3], v0, s[28:31], 0 offen
 ; CHECK-NEXT:    v_mfma_f32_32x32x16_bf16 v[16:31], a[16:19], a[16:19], v[16:31]
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[56:57], exec
 ; CHECK-NEXT:    s_cselect_b32 s52, s15, 0
@@ -155,7 +155,7 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    v_mov_b32_e32 v14, s53
 ; CHECK-NEXT:    v_mov_b32_e32 v15, s52
 ; CHECK-NEXT:    s_cselect_b32 s52, s65, 0
-; CHECK-NEXT:    s_cselect_b32 s53, s63, 0
+; CHECK-NEXT:    s_cselect_b32 s53, s64, 0
 ; CHECK-NEXT:    s_cselect_b32 s54, s62, 0
 ; CHECK-NEXT:    s_cselect_b32 s33, s33, 0
 ; CHECK-NEXT:    s_and_b64 s[50:51], s[50:51], exec
@@ -190,14 +190,14 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_mov_b64 s[58:59], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[56:57], -1
 ; CHECK-NEXT:    s_mov_b64 s[50:51], -1
-; CHECK-NEXT:    s_mov_b32 s64, 1
+; CHECK-NEXT:    s_mov_b32 s63, 1
 ; CHECK-NEXT:    s_mov_b64 s[54:55], 0
 ; CHECK-NEXT:    s_mov_b64 s[48:49], s[34:35]
 ; CHECK-NEXT:    s_mov_b64 s[46:47], s[36:37]
 ; CHECK-NEXT:    s_mov_b32 s21, s27
 ; CHECK-NEXT:    s_mov_b32 s33, 1
 ; CHECK-NEXT:    s_mov_b32 s62, 1
-; CHECK-NEXT:    s_mov_b32 s63, 1
+; CHECK-NEXT:    s_mov_b32 s64, 1
 ; CHECK-NEXT:    s_mov_b32 s65, 1
 ; CHECK-NEXT:    ds_write_b64 v4, v[12:13]
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB0_1
@@ -207,9 +207,9 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_cselect_b32 s4, 0x3f803f80, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v4, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v5, s4
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v6, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v7, s4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    ds_write_b128 v0, v[4:7]
 ; CHECK-NEXT:    v_mov_b32_e32 v4, s23



More information about the llvm-branch-commits mailing list