[llvm] d9bcffd - [AMDGPU] Add AGPR pressure tracking (#214745)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 03:31:53 PDT 2026
Author: Romanov Vlad
Date: 2026-09-07T12:31:48+02:00
New Revision: d9bcffd284812c3e29805af27d32fdedd82c5acc
URL: https://github.com/llvm/llvm-project/commit/d9bcffd284812c3e29805af27d32fdedd82c5acc
DIFF: https://github.com/llvm/llvm-project/commit/d9bcffd284812c3e29805af27d32fdedd82c5acc.diff
LOG: [AMDGPU] Add AGPR pressure tracking (#214745)
Add AGPR pressure tacking in RPDelta.
Added:
llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
Modified:
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
llvm/test/CodeGen/AMDGPU/agpr-csr.ll
llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
llvm/test/CodeGen/AMDGPU/spill-agpr.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 4d67ba9f1d8d7..9ec2a60ec3a8e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -893,14 +893,17 @@ void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
unsigned SGPRPressure = 0;
unsigned VGPRPressure = 0;
+ unsigned AGPRPressure = 0;
PickedPending = false;
if (DAG->isTrackingPressure()) {
if (!useGCNTrackers()) {
SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
+ AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
} else {
SGPRPressure = DownwardTracker.getPressure().getSGPRNum();
VGPRPressure = DownwardTracker.getPressure().getArchVGPRNum();
+ AGPRPressure = DownwardTracker.getPressure().getAGPRNum();
}
}
@@ -908,7 +911,7 @@ void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
for (SUnit *SU : Q) {
SchedCandidate TryCand(ZonePolicy);
initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, SGPRPressure,
- VGPRPressure, IsBottomUp);
+ VGPRPressure, AGPRPressure, IsBottomUp);
SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
tryCandidateCoexec(Cand, TryCand, ZoneArg);
if (TryCand.Reason != NoCand) {
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 3dd468772e9b1..86ea8b0a98041 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -150,6 +150,8 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass);
VGPRExcessLimit =
Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass);
+ AGPRExcessLimit =
+ Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::AGPR_32RegClass);
SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
// Set the initial TargetOccupnacy to the maximum occupancy that we can
@@ -181,6 +183,10 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
VGPRBudget = std::max(VGPRBudget, Granule);
VGPRCriticalLimit = std::min(VGPRBudget, VGPRExcessLimit);
}
+
+ // Reuse VGPR critical limit
+ AGPRCriticalLimit = std::min(VGPRCriticalLimit, AGPRExcessLimit);
+
// Apply VGPR excess threshold percentage if specified.
if (VGPRThresholdPercentOpt > 0) {
[[maybe_unused]] unsigned OriginalVGPRExcessLimit = VGPRExcessLimit;
@@ -202,8 +208,14 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) {
// Subtract error margin and bias from register limits and avoid overflow.
SGPRCriticalLimit -= std::min(SGPRLimitBias + ErrorMargin, SGPRCriticalLimit);
SGPRExcessLimit -= std::min(SGPRLimitBias + ErrorMargin, SGPRExcessLimit);
+
+ AGPRExcessLimit -= std::min(VGPRLimitBias + ErrorMargin, AGPRExcessLimit);
+ AGPRCriticalLimit -= std::min(VGPRLimitBias + ErrorMargin, AGPRCriticalLimit);
+
LLVM_DEBUG(dbgs() << "VGPRCriticalLimit = " << VGPRCriticalLimit
<< ", VGPRExcessLimit = " << VGPRExcessLimit
+ << ", AGPRCriticalLimit = " << AGPRCriticalLimit
+ << ", AGPRExcessLimit = " << AGPRExcessLimit
<< ", SGPRCriticalLimit = " << SGPRCriticalLimit
<< ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n");
}
@@ -315,7 +327,8 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
const RegPressureTracker &RPTracker,
const SIRegisterInfo *SRI,
unsigned SGPRPressure,
- unsigned VGPRPressure, bool IsBottomUp) {
+ unsigned VGPRPressure,
+ unsigned AGPRPressure, bool IsBottomUp) {
Cand.SU = SU;
Cand.AtTop = AtTop;
@@ -344,6 +357,7 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
Pressure.resize(4, 0);
Pressure[AMDGPU::RegisterPressureSets::SReg_32] = SGPRPressure;
Pressure[AMDGPU::RegisterPressureSets::VGPR_32] = VGPRPressure;
+ Pressure[AMDGPU::RegisterPressureSets::AGPR_32] = AGPRPressure;
for (const auto &Diff : DAG->getPressureDiff(SU)) {
if (!Diff.isValid())
@@ -361,7 +375,9 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
if (Pressure[AMDGPU::RegisterPressureSets::SReg_32] !=
CheckPressure[AMDGPU::RegisterPressureSets::SReg_32] ||
Pressure[AMDGPU::RegisterPressureSets::VGPR_32] !=
- CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32]) {
+ CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] ||
+ Pressure[AMDGPU::RegisterPressureSets::AGPR_32] !=
+ CheckPressure[AMDGPU::RegisterPressureSets::AGPR_32]) {
errs() << "Register Pressure is inaccurate when calculated through "
"PressureDiff\n"
<< "SGPR got " << Pressure[AMDGPU::RegisterPressureSets::SReg_32]
@@ -369,12 +385,16 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
<< CheckPressure[AMDGPU::RegisterPressureSets::SReg_32] << "\n"
<< "VGPR got " << Pressure[AMDGPU::RegisterPressureSets::VGPR_32]
<< ", expected "
- << CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] << "\n";
+ << CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] << "\n"
+ << "AGPR got " << Pressure[AMDGPU::RegisterPressureSets::AGPR_32]
+ << ", expected "
+ << CheckPressure[AMDGPU::RegisterPressureSets::AGPR_32] << "\n";
report_fatal_error("inaccurate register pressure calculation");
}
#endif
}
+ unsigned NewAGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
unsigned NewSGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
unsigned NewVGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
@@ -383,18 +403,19 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
// instruction that increases the set with the least amount of registers,
// which in our case would be SGPRs. This is rarely what we want, so
// when we report excess/critical register pressure, we do it either
- // only for VGPRs or only for SGPRs.
+ // only for VGPRs, AGPRs or SGPRs. Priority: VGPR > AGPR > SGPR.
// FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
const unsigned MaxVGPRPressureInc = 16;
bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
- bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
-
+ bool ShouldTrackAGPRs = AGPRExcessLimit > 0 && !ShouldTrackVGPRs &&
+ AGPRPressure + MaxVGPRPressureInc >= AGPRExcessLimit;
+ bool ShouldTrackSGPRs =
+ !ShouldTrackVGPRs && !ShouldTrackAGPRs && SGPRPressure >= SGPRExcessLimit;
// FIXME: We have to enter REG-EXCESS before we reach the actual threshold
// to increase the likelihood we don't go over the limits. We should improve
// the analysis to look through dependencies to find the path with the least
// register pressure.
-
// We only need to update the RPDelta for instructions that increase register
// pressure. Instructions that decrease or keep reg pressure the same will be
// marked as RegExcess in tryCandidate() when they are compared with
@@ -405,6 +426,12 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
}
+ if (ShouldTrackAGPRs && NewAGPRPressure >= AGPRExcessLimit) {
+ HasHighPressure = true;
+ Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::AGPR_32);
+ Cand.RPDelta.Excess.setUnitInc(NewAGPRPressure - AGPRExcessLimit);
+ }
+
if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
HasHighPressure = true;
Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
@@ -413,22 +440,29 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
// Register pressure is considered 'CRITICAL' if it is approaching a value
// that would reduce the wave occupancy for the execution unit. When
- // register pressure is 'CRITICAL', increasing SGPR and VGPR pressure both
- // has the same cost, so we don't need to prefer one over the other.
+ // register pressure is 'CRITICAL', increasing SGPR, VGPR, and AGPR
+ // pressure all has the same cost, so we pick the most critical type.
int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
+ int AGPRDelta = AGPRExcessLimit > 0 ? NewAGPRPressure - AGPRCriticalLimit
+ : std::numeric_limits<int>::min();
- if (SGPRDelta >= 0 || VGPRDelta >= 0) {
+ if (SGPRDelta >= 0 || VGPRDelta >= 0 || AGPRDelta >= 0) {
HasHighPressure = true;
- if (SGPRDelta > VGPRDelta) {
- Cand.RPDelta.CriticalMax =
- PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
- Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
- } else {
+ // Pick the most critical type.
+ if (VGPRDelta >= SGPRDelta && VGPRDelta >= AGPRDelta) {
Cand.RPDelta.CriticalMax =
PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
+ } else if (AGPRDelta >= SGPRDelta) {
+ Cand.RPDelta.CriticalMax =
+ PressureChange(AMDGPU::RegisterPressureSets::AGPR_32);
+ Cand.RPDelta.CriticalMax.setUnitInc(AGPRDelta);
+ } else {
+ Cand.RPDelta.CriticalMax =
+ PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
+ Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
}
}
}
@@ -478,17 +512,20 @@ void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
unsigned SGPRPressure = 0;
unsigned VGPRPressure = 0;
+ unsigned AGPRPressure = 0;
IsPending = false;
if (DAG->isTrackingPressure()) {
if (!useGCNTrackers()) {
SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
+ AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
} else {
GCNRPTracker *T = IsBottomUp
? static_cast<GCNRPTracker *>(&UpwardTracker)
: static_cast<GCNRPTracker *>(&DownwardTracker);
SGPRPressure = T->getPressure().getSGPRNum();
VGPRPressure = T->getPressure().getArchVGPRNum();
+ AGPRPressure = T->getPressure().getAGPRNum();
}
}
LLVM_DEBUG(dbgs() << "Available Q:\n");
@@ -497,7 +534,7 @@ void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
SchedCandidate TryCand(ZonePolicy);
initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, SGPRPressure,
- VGPRPressure, IsBottomUp);
+ VGPRPressure, AGPRPressure, IsBottomUp);
// Pass SchedBoundary only when comparing nodes from the same boundary.
SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
tryCandidate(Cand, TryCand, ZoneArg);
@@ -521,7 +558,7 @@ void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
SchedCandidate TryCand(ZonePolicy);
initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, SGPRPressure,
- VGPRPressure, IsBottomUp);
+ VGPRPressure, AGPRPressure, IsBottomUp);
// Pass SchedBoundary only when comparing nodes from the same boundary.
SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
tryPendingCandidate(Cand, TryCand, ZoneArg);
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2059f4e6479ff..29393d035cc8c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -57,7 +57,8 @@ class GCNSchedStrategy : public GenericScheduler {
void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop,
const RegPressureTracker &RPTracker,
const SIRegisterInfo *SRI, unsigned SGPRPressure,
- unsigned VGPRPressure, bool IsBottomUp);
+ unsigned VGPRPressure, unsigned AGPRPressure,
+ bool IsBottomUp);
/// Estimate how many cycles \p SU must wait due to structural hazards at the
/// current boundary cycle. Returns zero when no stall is required.
@@ -92,6 +93,8 @@ class GCNSchedStrategy : public GenericScheduler {
unsigned VGPRExcessLimit;
+ unsigned AGPRExcessLimit;
+
unsigned TargetOccupancy;
MachineFunction *MF;
@@ -136,6 +139,8 @@ class GCNSchedStrategy : public GenericScheduler {
unsigned VGPRCriticalLimit;
+ unsigned AGPRCriticalLimit;
+
unsigned SGPRLimitBias = 0;
unsigned VGPRLimitBias = 0;
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
index 016ac4eeb8867..c0805565190fb 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
@@ -180,34 +180,34 @@ define amdgpu_kernel void @test_call_empty() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a24
; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
@@ -216,19 +216,19 @@ define amdgpu_kernel void @test_call_empty() #0 {
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -315,34 +315,34 @@ define amdgpu_kernel void @test_call_areg4() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a24
; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
@@ -351,19 +351,19 @@ define amdgpu_kernel void @test_call_areg4() #0 {
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -450,34 +450,34 @@ define amdgpu_kernel void @test_call_areg32() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a24
; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
@@ -486,19 +486,19 @@ define amdgpu_kernel void @test_call_areg32() #0 {
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -585,34 +585,34 @@ define amdgpu_kernel void @test_call_areg64() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a24
; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
@@ -621,19 +621,19 @@ define amdgpu_kernel void @test_call_areg64() #0 {
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -720,34 +720,34 @@ define amdgpu_kernel void @test_call_areg31_63() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v30, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a24
; GFX908-NEXT: v_accvgpr_read_b32 v35, a31
; GFX908-NEXT: v_accvgpr_read_b32 v34, a30
; GFX908-NEXT: v_accvgpr_read_b32 v33, a29
@@ -756,19 +756,19 @@ define amdgpu_kernel void @test_call_areg31_63() #0 {
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_endpgm
bb:
@@ -856,30 +856,30 @@ define amdgpu_kernel void @test_call_unknown() #0 {
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: v_accvgpr_read_b32 v43, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v47, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v59, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v63, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v75, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v79, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v91, a27
; GFX908-NEXT: v_accvgpr_read_b32 v42, a2
; GFX908-NEXT: v_accvgpr_read_b32 v41, a1
; GFX908-NEXT: v_accvgpr_read_b32 v40, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v47, a7
; GFX908-NEXT: v_accvgpr_read_b32 v46, a6
; GFX908-NEXT: v_accvgpr_read_b32 v45, a5
; GFX908-NEXT: v_accvgpr_read_b32 v44, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v59, a11
; GFX908-NEXT: v_accvgpr_read_b32 v58, a10
; GFX908-NEXT: v_accvgpr_read_b32 v57, a9
; GFX908-NEXT: v_accvgpr_read_b32 v56, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v63, a15
; GFX908-NEXT: v_accvgpr_read_b32 v62, a14
; GFX908-NEXT: v_accvgpr_read_b32 v61, a13
; GFX908-NEXT: v_accvgpr_read_b32 v60, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v75, a19
; GFX908-NEXT: v_accvgpr_read_b32 v74, a18
; GFX908-NEXT: v_accvgpr_read_b32 v73, a17
; GFX908-NEXT: v_accvgpr_read_b32 v72, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v79, a23
; GFX908-NEXT: v_accvgpr_read_b32 v78, a22
; GFX908-NEXT: v_accvgpr_read_b32 v77, a21
; GFX908-NEXT: v_accvgpr_read_b32 v76, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v91, a27
; GFX908-NEXT: v_accvgpr_read_b32 v90, a26
; GFX908-NEXT: v_accvgpr_read_b32 v89, a25
; GFX908-NEXT: v_accvgpr_read_b32 v88, a24
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll
index c8f75373a55dd..1a3f1ef9223ae 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll
@@ -34,7 +34,7 @@ define void @func_illegal_agpr_use_asm() #0 {
; GFX908: v_accvgpr_write_b32
; GFX90A-NOT: v_accvgpr_write_b32
-; GFX908: NumVgprs: 5
+; GFX908: NumVgprs: 30
; GFX908: NumAgprs: 32
; GFX90A: NumVgprs: 35
; GFX90A: NumAgprs: 0
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
new file mode 100644
index 0000000000000..7e65714c45d18
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
@@ -0,0 +1,71 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -amdgpu-stress-agpr=8 %s -o - | FileCheck --check-prefix=LIMIT %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck --check-prefix=NO_LIMIT %s
+
+# Test that AGPR pressure is tracked during scheduling.
+# Two independent MFMA chains (3 MFMAs each), results written via DS_WRITE.
+# With -amdgpu-stress-agpr=8, the scheduler should prefer scheduling
+# one chain's consumer before starting the other chain to limit AGPR pressure.
+
+--- |
+ define void @agpr_excess_tracking() #0 { ret void }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+name: agpr_excess_tracking
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; LIMIT-LABEL: name: agpr_excess_tracking
+ ; LIMIT: %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ ; LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ ; LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
+ ; LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
+ ; LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; LIMIT-NEXT: S_ENDPGM 0
+ ;
+ ; NO_LIMIT-LABEL: name: agpr_excess_tracking
+ ; NO_LIMIT: %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
+ ; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ ; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
+ ; NO_LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: S_ENDPGM 0
+ %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3)
+ %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
+ %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3)
+
+ %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3)
+ %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3)
+
+ ; Chain 0
+ %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
+ %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; Chain 1
+ %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
+ %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
+
+ DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
index 5810d859f0cf4..ef04672556acc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
@@ -32,6 +32,77 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; SDAG-NEXT: ds_read_b128 a[8:11], v2 offset:57376
; SDAG-NEXT: ds_read_b128 a[12:15], v2 offset:57392
; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
+; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[32:35], v1
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:8304
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:8288
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:8272
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:8256
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:8240
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:8224
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:8208
+; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:8192
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; SDAG-NEXT: s_nop 9
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:24688
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:24672
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:24656
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:24640
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:24624
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:24608
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:24592
+; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:24576
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16400
; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:49264
; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:49248
; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:49232
@@ -40,222 +111,149 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:49184
; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:49168
; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:49152
-; SDAG-NEXT: s_waitcnt lgkmcnt(8)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; SDAG-NEXT: ds_read_b128 a[156:159], v1 offset:112
-; SDAG-NEXT: ds_read_b128 a[152:155], v1 offset:96
-; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
-; SDAG-NEXT: s_waitcnt lgkmcnt(4)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
-; SDAG-NEXT: ds_read_b128 a[148:151], v1 offset:80
-; SDAG-NEXT: ds_read_b128 a[144:147], v1 offset:64
-; SDAG-NEXT: ds_read_b128 a[128:131], v1
-; SDAG-NEXT: ds_read_b128 a[132:135], v1 offset:16
-; SDAG-NEXT: ds_read_b128 a[136:139], v1 offset:32
-; SDAG-NEXT: ds_read_b128 a[140:143], v1 offset:48
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159]
-; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:8304
-; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:8288
-; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:8272
-; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:8256
-; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:8240
-; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:8224
-; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:8208
-; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:8192
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127]
-; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
-; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:112
-; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:96
-; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:80
-; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:64
-; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:48
-; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32
-; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:16
-; SDAG-NEXT: ds_write_b128 v0, a[128:131]
-; SDAG-NEXT: v_mov_b32_e32 v0, s1
-; SDAG-NEXT: s_waitcnt lgkmcnt(8)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:24672
; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:24688
; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:24640
-; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:8288
-; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:8304
-; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:8256
-; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:8272
-; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:8224
-; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:8240
-; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:8192
-; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:8208
; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:24656
; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:24608
; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:24624
; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:24576
; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:24592
-; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
-; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
-; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
-; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
-; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
-; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
-; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768
-; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_iglp_opt_mfma_gemm:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: v_lshlrev_b32_e32 v33, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v68, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v69, 2.0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v0, s0, v33
-; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:8208
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:8304
+; GISEL-NEXT: v_add_u32_e32 v70, s0, v0
+; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v70
+; GISEL-NEXT: ds_read_b128 a[0:3], v1 offset:32768
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:32784
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32800
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:32816
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:32832
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:32848
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:32864
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:32880
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v32, v34, a[96:127]
-; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:24592
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:24688
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: ds_read_b128 a[152:155], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[156:159], v0 offset:112
-; GISEL-NEXT: v_add_u32_e32 v33, s1, v33
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v68, v69, a[0:31]
+; GISEL-NEXT: ds_read_b128 a[32:35], v70
+; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:16
+; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:32
+; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:48
+; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:64
+; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:80
+; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:96
+; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:112
+; GISEL-NEXT: v_add_u32_e32 v71, s1, v0
; GISEL-NEXT: ; iglp_opt mask(0x00000000)
-; GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v32, v34, a[0:31]
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:49264
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v32, v34, a[32:63]
-; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:32784
-; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:32816
-; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:32880
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v32, v34, a[64:95]
-; GISEL-NEXT: ds_read_b128 a[128:131], v0
-; GISEL-NEXT: ds_read_b128 a[132:135], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[136:139], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[140:143], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[144:147], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[148:151], v0 offset:80
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63]
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 2
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a32
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v32, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v33, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v34, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v35, a63
+; GISEL-NEXT: ds_write_b128 v71, v[4:7]
+; GISEL-NEXT: ds_write_b128 v71, v[8:11] offset:16
+; GISEL-NEXT: ds_write_b128 v71, v[12:15] offset:32
+; GISEL-NEXT: ds_write_b128 v71, v[16:19] offset:48
+; GISEL-NEXT: ds_write_b128 v71, v[20:23] offset:64
+; GISEL-NEXT: ds_write_b128 v71, v[24:27] offset:80
+; GISEL-NEXT: ds_write_b128 v71, v[28:31] offset:96
+; GISEL-NEXT: ds_read_b128 a[32:35], v70 offset:8192
+; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:8208
+; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:8224
+; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:8240
+; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:8256
+; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:8272
+; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:8288
+; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:8304
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v32, v34, a[128:159]
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63]
+; GISEL-NEXT: ds_write_b128 v71, v[32:35] offset:112
; GISEL-NEXT: v_mov_b32_e32 v32, s1
; GISEL-NEXT: s_nop 15
-; GISEL-NEXT: s_nop 1
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ds_write_b128 v33, v[0:3]
-; GISEL-NEXT: ds_write_b128 v33, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v33, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v33, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v33, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v33, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v33, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v33, v[28:31] offset:112
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
+; GISEL-NEXT: ds_read_b128 a[32:35], v70 offset:24576
+; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:24592
+; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:24608
+; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:24624
+; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:24640
+; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:24656
+; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:24672
+; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:24688
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63]
; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
@@ -275,6 +273,48 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v67, a63
+; GISEL-NEXT: v_accvgpr_read_b32 v66, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v65, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v64, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v63, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v62, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v61, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v60, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v59, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v58, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v57, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v56, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v55, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v54, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v53, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v52, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v51, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v50, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v49, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v48, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v47, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v46, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v45, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v44, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v43, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v42, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v41, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v40, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v39, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v38, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v37, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v36, a32
+; GISEL-NEXT: ds_read_b128 a[32:35], v70 offset:49152
+; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:49168
+; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:49184
+; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:49200
+; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:49216
+; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:49232
+; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:49248
+; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:49264
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63]
; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
@@ -294,88 +334,56 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v99, a63
+; GISEL-NEXT: v_accvgpr_read_b32 v98, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v97, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v96, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v95, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v94, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v93, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v92, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v91, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v90, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v89, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v88, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v87, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v86, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v85, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v84, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v83, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v82, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v81, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v80, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v79, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v78, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v77, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v76, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v75, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v74, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v73, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v72, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v71, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v70, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v69, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v68, a32
; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
+; GISEL-NEXT: ds_write_b128 v32, v[36:39] offset:16384
+; GISEL-NEXT: ds_write_b128 v32, v[40:43] offset:16400
+; GISEL-NEXT: ds_write_b128 v32, v[44:47] offset:16416
+; GISEL-NEXT: ds_write_b128 v32, v[48:51] offset:16432
+; GISEL-NEXT: ds_write_b128 v32, v[52:55] offset:16448
+; GISEL-NEXT: ds_write_b128 v32, v[56:59] offset:16464
+; GISEL-NEXT: ds_write_b128 v32, v[60:63] offset:16480
+; GISEL-NEXT: ds_write_b128 v32, v[64:67] offset:16496
+; GISEL-NEXT: ds_write_b128 v32, v[68:71] offset:24576
+; GISEL-NEXT: ds_write_b128 v32, v[72:75] offset:24592
+; GISEL-NEXT: ds_write_b128 v32, v[76:79] offset:24608
+; GISEL-NEXT: ds_write_b128 v32, v[80:83] offset:24624
+; GISEL-NEXT: ds_write_b128 v32, v[84:87] offset:24640
+; GISEL-NEXT: ds_write_b128 v32, v[88:91] offset:24656
+; GISEL-NEXT: ds_write_b128 v32, v[92:95] offset:24672
+; GISEL-NEXT: ds_write_b128 v32, v[96:99] offset:24688
; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
@@ -527,42 +535,28 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: v_lshlrev_b32_e32 v96, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v97, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v98, 2.0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v0, s0, v32
-; GISEL-NEXT: ds_read_b128 a[0:3], v0
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:80
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; GISEL-NEXT: v_add_u32_e32 v64, s0, v96
+; GISEL-NEXT: ds_read_b128 a[0:3], v64
+; GISEL-NEXT: ds_read_b128 a[4:7], v64 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v64 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v64 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v64 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v64 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v64 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v64 offset:112
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:8208
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:8304
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: v_add_u32_e32 v32, s1, v32
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31]
+; GISEL-NEXT: v_add_u32_e32 v99, 0xc000, v64
+; GISEL-NEXT: v_add_u32_e32 v96, s1, v96
; GISEL-NEXT: ; iglp_opt mask(0x00000001)
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63]
-; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:24592
-; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:24688
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
@@ -570,42 +564,18 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95]
-; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127]
-; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
-; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
-; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
@@ -617,14 +587,122 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: ds_write_b128 v32, v[0:3]
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
+; GISEL-NEXT: ds_read_b128 a[0:3], v64 offset:8192
+; GISEL-NEXT: ds_read_b128 a[4:7], v64 offset:8208
+; GISEL-NEXT: ds_read_b128 a[8:11], v64 offset:8224
+; GISEL-NEXT: ds_read_b128 a[12:15], v64 offset:8240
+; GISEL-NEXT: ds_read_b128 a[16:19], v64 offset:8256
+; GISEL-NEXT: ds_read_b128 a[20:23], v64 offset:8272
+; GISEL-NEXT: ds_read_b128 a[24:27], v64 offset:8288
+; GISEL-NEXT: ds_read_b128 a[28:31], v64 offset:8304
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31]
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 2
+; GISEL-NEXT: v_accvgpr_read_b32 v63, a31
+; GISEL-NEXT: v_accvgpr_read_b32 v62, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v61, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v60, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v59, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v58, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v57, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v56, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v55, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v54, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v53, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v52, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v51, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v50, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v49, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v48, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v47, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v46, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v45, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v44, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v43, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v42, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v41, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v40, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v39, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v38, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v37, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v36, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v35, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v34, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v33, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v32, a0
+; GISEL-NEXT: ds_read_b128 a[0:3], v64 offset:24576
+; GISEL-NEXT: ds_read_b128 a[4:7], v64 offset:24592
+; GISEL-NEXT: ds_read_b128 a[8:11], v64 offset:24608
+; GISEL-NEXT: ds_read_b128 a[12:15], v64 offset:24624
+; GISEL-NEXT: ds_read_b128 a[16:19], v64 offset:24640
+; GISEL-NEXT: ds_read_b128 a[20:23], v64 offset:24656
+; GISEL-NEXT: ds_read_b128 a[24:27], v64 offset:24672
+; GISEL-NEXT: ds_read_b128 a[28:31], v64 offset:24688
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31]
+; GISEL-NEXT: ds_read_b128 a[32:35], v64 offset:49152
+; GISEL-NEXT: ds_read_b128 a[36:39], v64 offset:49168
+; GISEL-NEXT: ds_read_b128 a[40:43], v64 offset:49184
+; GISEL-NEXT: ds_read_b128 a[44:47], v64 offset:49200
+; GISEL-NEXT: ds_read_b128 a[48:51], v64 offset:49216
+; GISEL-NEXT: ds_read_b128 a[52:55], v64 offset:49232
+; GISEL-NEXT: ds_read_b128 a[56:59], v64 offset:49248
+; GISEL-NEXT: ds_read_b128 a[60:63], v64 offset:49264
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v97, v98, a[32:63]
+; GISEL-NEXT: s_nop 8
+; GISEL-NEXT: v_accvgpr_read_b32 v95, a31
+; GISEL-NEXT: v_accvgpr_read_b32 v94, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v93, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v92, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v91, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v90, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v89, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v88, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v87, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v86, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v85, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v84, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v83, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v82, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v81, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v80, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v79, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v78, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v77, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v76, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v75, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v74, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v73, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v72, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v71, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v70, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v69, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v68, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v67, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v66, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v65, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v64, a0
+; GISEL-NEXT: ds_read_b128 a[0:3], v99 offset:32768
+; GISEL-NEXT: ds_read_b128 a[4:7], v99 offset:32784
+; GISEL-NEXT: ds_read_b128 a[8:11], v99 offset:32800
+; GISEL-NEXT: ds_read_b128 a[12:15], v99 offset:32816
+; GISEL-NEXT: ds_read_b128 a[16:19], v99 offset:32832
+; GISEL-NEXT: ds_read_b128 a[20:23], v99 offset:32848
+; GISEL-NEXT: ds_read_b128 a[24:27], v99 offset:32864
+; GISEL-NEXT: ds_read_b128 a[28:31], v99 offset:32880
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31]
+; GISEL-NEXT: ds_write_b128 v96, v[0:3]
+; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:112
+; GISEL-NEXT: v_mov_b32_e32 v96, s1
; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
@@ -657,135 +735,70 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: v_mov_b32_e32 v32, s1
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: ds_write_b128 v96, v[32:35] offset:8192
+; GISEL-NEXT: ds_write_b128 v96, v[36:39] offset:8208
+; GISEL-NEXT: ds_write_b128 v96, v[40:43] offset:8224
+; GISEL-NEXT: ds_write_b128 v96, v[44:47] offset:8240
+; GISEL-NEXT: ds_write_b128 v96, v[48:51] offset:8256
+; GISEL-NEXT: ds_write_b128 v96, v[52:55] offset:8272
+; GISEL-NEXT: ds_write_b128 v96, v[56:59] offset:8288
+; GISEL-NEXT: ds_write_b128 v96, v[60:63] offset:8304
+; GISEL-NEXT: ds_write_b128 v96, v[64:67] offset:16384
+; GISEL-NEXT: ds_write_b128 v96, v[68:71] offset:16400
+; GISEL-NEXT: ds_write_b128 v96, v[72:75] offset:16416
+; GISEL-NEXT: ds_write_b128 v96, v[76:79] offset:16432
+; GISEL-NEXT: ds_write_b128 v96, v[80:83] offset:16448
+; GISEL-NEXT: ds_write_b128 v96, v[84:87] offset:16464
+; GISEL-NEXT: ds_write_b128 v96, v[88:91] offset:16480
+; GISEL-NEXT: ds_write_b128 v96, v[92:95] offset:16496
+; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:24688
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:32880
; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.iglp.opt(i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
index 279de9b7899e0..c296e5615d4ae 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
@@ -17,7 +17,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-LABEL: test_mfma_f32_32x32x2bf16:
; GFX908-SDAG: ; %bb.0: ; %bb
; GFX908-SDAG-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v29, 0
; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-SDAG-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -27,13 +27,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s18
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s22
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s23
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s21
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s22
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s23
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v0
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v1
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1
; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s24
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s25
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s26
@@ -74,7 +72,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s10
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s11
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s12
-; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s20
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, s20
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s21
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v3
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a26, v0
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a27, v1
@@ -83,7 +82,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s14
; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s15
; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, 1
-; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v5
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v5
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a29, v0
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a30, v1
; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a31, v2
@@ -96,57 +96,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a28
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a16
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a20
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a8
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; GFX908-SDAG-NEXT: s_nop 0
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-SDAG-NEXT: s_nop 1
-; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a19
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; GFX908-SDAG-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 086fc6f0ed4d7..4c13ebc3d590e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -29,7 +29,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -39,13 +39,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s18
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s23
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s21
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s22
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s23
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s24
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s25
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s26
@@ -86,7 +84,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s10
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s12
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s21
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
@@ -95,7 +94,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
@@ -108,57 +108,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32:
@@ -284,7 +269,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -294,13 +279,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s18
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s23
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s21
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s22
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s23
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s24
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s25
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s26
@@ -341,7 +324,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s10
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s12
-; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s21
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
@@ -350,7 +334,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
@@ -363,57 +348,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x1f32:
@@ -1833,7 +1803,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x4f16:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v32, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40
@@ -1842,11 +1812,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s17
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s18
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s24
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s23
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s24
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s25
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v0
@@ -1891,8 +1859,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s12
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s13
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s21
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s21
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s22
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0
@@ -1902,8 +1871,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v6
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s2
@@ -1916,57 +1886,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:112
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:64
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:80
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:32
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:48
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37]
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a4
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37] offset:96
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:112
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4f16:
@@ -2095,7 +2050,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-LABEL: test_mfma_f32_32x32x4f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v32, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40
@@ -2104,11 +2059,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s17
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s18
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s24
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s23
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s24
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s25
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v0
@@ -2153,8 +2106,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s12
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s13
-; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s20
-; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s21
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s21
+; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s22
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0
@@ -2164,8 +2118,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s0
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v6
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s2
@@ -2178,57 +2133,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:96
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:112
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:64
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:80
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:32
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:48
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37]
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a4
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37] offset:96
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:112
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37]
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x4f16:
@@ -3758,7 +3698,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-LABEL: test_mfma_i32_32x32x4i8:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v28, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -3768,13 +3708,11 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s18
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s23
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s21
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s22
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s23
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s24
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s25
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s26
@@ -3816,6 +3754,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s12
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s21
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
@@ -3825,6 +3764,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
@@ -3833,47 +3773,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a25
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:96
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a31
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a23
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a19
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:96
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a29
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a28
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:112
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:64
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a14
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a13
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:80
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a10
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a8
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[34:35] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[34:35]
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:112
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[20:23], s[34:35] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[16:19], s[34:35] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[12:15], s[34:35] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[8:11], s[34:35] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[4:7], s[34:35]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_i32_32x32x4i8:
@@ -3999,7 +3939,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-LABEL: test_mfma_i32_32x32x4i8:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v28, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -4009,13 +3949,11 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s18
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22
+; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s23
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s21
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s22
-; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s23
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s24
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s25
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s26
@@ -4057,6 +3995,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s11
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s12
; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20
+; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s21
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1
@@ -4066,6 +4005,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, 1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
@@ -4074,47 +4014,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; LIT-SRCC-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a25
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:96
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a31
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a23
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a19
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:96
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a29
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a28
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:112
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:64
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a14
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a13
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:80
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a10
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a8
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[34:35] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[34:35]
-; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a16
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:112
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[20:23], s[34:35] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[16:19], s[34:35] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[12:15], s[34:35] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[8:11], s[34:35] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[4:7], s[34:35]
+; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[0:3], s[34:35] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_i32_32x32x4i8:
@@ -5672,6 +5612,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -5734,75 +5675,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v2
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s13
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s14
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 2.0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
-; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
-; NOLIT-SRCC-NEXT: s_nop 15
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 2.0
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
+; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
+; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
+; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
@@ -5929,6 +5854,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -5999,8 +5925,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 2.0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
-; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; LIT-SRCC-NEXT: s_nop 15
@@ -6009,57 +5934,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
@@ -7880,7 +7790,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
@@ -7888,58 +7798,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
@@ -8031,7 +7926,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v14, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, 0
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
@@ -8039,47 +7934,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a25
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a24
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:112
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[4:7], s[0:1] offset:96
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[8:11], s[0:1] offset:80
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a18
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:48
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[10:13], s[0:1] offset:64
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
@@ -8841,7 +8732,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
@@ -8849,58 +8740,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
@@ -9024,7 +8900,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0
; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
@@ -9032,58 +8908,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x1f32_imm:
@@ -9940,43 +9801,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19
; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
@@ -10130,43 +9990,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19
; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x1f32_vecarg:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 96edd7876e981..5d20c7c874760 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -863,23 +863,23 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; SDAG-NEXT: v_mov_b32_e32 v1, 2.0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_add_u32_e32 v3, s0, v0
-; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:112
-; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:96
-; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:80
-; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:64
-; SDAG-NEXT: ds_read_b128 a[32:35], v3
-; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:16
-; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:32
-; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:48
+; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; SDAG-NEXT: ds_read_b128 a[0:3], v3
+; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:48
; SDAG-NEXT: v_add_u32_e32 v4, 0x6000, v3
-; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:8304
-; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:8288
-; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:8272
-; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:8256
-; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:8240
-; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:8224
-; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:8208
-; SDAG-NEXT: ds_read_b128 a[0:3], v3 offset:8192
+; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; SDAG-NEXT: ds_read_b128 a[32:35], v3 offset:8192
; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:24688
; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:24672
; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:24656
@@ -905,33 +905,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; SDAG-NEXT: ds_read_b128 a[136:139], v4 offset:57376
; SDAG-NEXT: ds_read_b128 a[140:143], v4 offset:57392
; SDAG-NEXT: s_waitcnt lgkmcnt(14)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
; SDAG-NEXT: s_waitcnt lgkmcnt(8)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[0:3]
; SDAG-NEXT: v_mov_b32_e32 v0, s1
-; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208
; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
@@ -964,145 +964,209 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v96, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v97, 2.0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_add_u32_e32 v0, s0, v32
-; GISEL-NEXT: ds_read_b128 a[96:99], v0
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:80
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:112
-; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:49264
-; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
-; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
-; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
+; GISEL-NEXT: v_add_u32_e32 v1, s0, v0
+; GISEL-NEXT: ds_read_b128 a[0:3], v1
+; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; GISEL-NEXT: v_add_u32_e32 v2, 0xc000, v1
+; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:24576
+; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:24592
+; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:24608
+; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:24624
+; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:24640
+; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:24656
+; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:24672
+; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:24688
+; GISEL-NEXT: ds_read_b128 a[96:99], v1 offset:8192
+; GISEL-NEXT: ds_read_b128 a[100:103], v1 offset:8208
+; GISEL-NEXT: ds_read_b128 a[104:107], v1 offset:8224
+; GISEL-NEXT: ds_read_b128 a[108:111], v1 offset:8240
+; GISEL-NEXT: ds_read_b128 a[112:115], v1 offset:8256
+; GISEL-NEXT: ds_read_b128 a[116:119], v1 offset:8272
+; GISEL-NEXT: ds_read_b128 a[120:123], v1 offset:8288
+; GISEL-NEXT: ds_read_b128 a[124:127], v1 offset:8304
+; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:49152
+; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:49168
+; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:49184
+; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:49200
+; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:49216
+; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:49232
+; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:49248
+; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:49264
+; GISEL-NEXT: ds_read_b128 a[32:35], v2 offset:32768
+; GISEL-NEXT: ds_read_b128 a[36:39], v2 offset:32784
+; GISEL-NEXT: ds_read_b128 a[40:43], v2 offset:32800
+; GISEL-NEXT: ds_read_b128 a[44:47], v2 offset:32816
+; GISEL-NEXT: ds_read_b128 a[48:51], v2 offset:32832
+; GISEL-NEXT: ds_read_b128 a[52:55], v2 offset:32848
+; GISEL-NEXT: ds_read_b128 a[56:59], v2 offset:32864
+; GISEL-NEXT: ds_read_b128 a[60:63], v2 offset:32880
; GISEL-NEXT: s_waitcnt lgkmcnt(14)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127]
-; GISEL-NEXT: v_add_u32_e32 v32, s1, v32
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v96, v97, a[0:31]
+; GISEL-NEXT: v_add_u32_e32 v98, s1, v0
; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v96, v97, a[128:159]
; GISEL-NEXT: s_nop 15
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63]
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
+; GISEL-NEXT: v_accvgpr_read_b32 v95, a31
+; GISEL-NEXT: v_accvgpr_read_b32 v94, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v93, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v92, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v91, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v90, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v89, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v88, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v87, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v86, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v85, a21
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v96, v97, a[96:127]
+; GISEL-NEXT: v_accvgpr_read_b32 v84, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v83, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v82, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v81, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v80, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v79, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v78, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v77, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v76, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v75, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v74, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v73, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v72, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v71, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v70, a6
; GISEL-NEXT: s_waitcnt lgkmcnt(8)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95]
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v96, v97, a[64:95]
+; GISEL-NEXT: v_accvgpr_read_b32 v69, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v68, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v67, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v66, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v65, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v64, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
-; GISEL-NEXT: ds_write_b128 v32, v[0:3]
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
-; GISEL-NEXT: v_mov_b32_e32 v32, s1
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v96, v97, a[32:63]
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
+; GISEL-NEXT: v_mov_b32_e32 v96, s1
+; GISEL-NEXT: v_accvgpr_read_b32 v32, a96
+; GISEL-NEXT: v_accvgpr_read_b32 v33, a97
+; GISEL-NEXT: v_accvgpr_read_b32 v34, a98
+; GISEL-NEXT: v_accvgpr_read_b32 v35, a99
+; GISEL-NEXT: v_accvgpr_read_b32 v36, a100
+; GISEL-NEXT: v_accvgpr_read_b32 v37, a101
+; GISEL-NEXT: v_accvgpr_read_b32 v38, a102
+; GISEL-NEXT: v_accvgpr_read_b32 v39, a103
+; GISEL-NEXT: v_accvgpr_read_b32 v40, a104
+; GISEL-NEXT: v_accvgpr_read_b32 v41, a105
+; GISEL-NEXT: v_accvgpr_read_b32 v42, a106
+; GISEL-NEXT: v_accvgpr_read_b32 v43, a107
+; GISEL-NEXT: v_accvgpr_read_b32 v44, a108
+; GISEL-NEXT: v_accvgpr_read_b32 v45, a109
+; GISEL-NEXT: v_accvgpr_read_b32 v46, a110
+; GISEL-NEXT: v_accvgpr_read_b32 v47, a111
+; GISEL-NEXT: v_accvgpr_read_b32 v48, a112
+; GISEL-NEXT: v_accvgpr_read_b32 v49, a113
+; GISEL-NEXT: v_accvgpr_read_b32 v50, a114
+; GISEL-NEXT: v_accvgpr_read_b32 v51, a115
+; GISEL-NEXT: v_accvgpr_read_b32 v52, a116
+; GISEL-NEXT: v_accvgpr_read_b32 v53, a117
+; GISEL-NEXT: v_accvgpr_read_b32 v54, a118
+; GISEL-NEXT: v_accvgpr_read_b32 v55, a119
+; GISEL-NEXT: v_accvgpr_read_b32 v56, a120
+; GISEL-NEXT: v_accvgpr_read_b32 v57, a121
+; GISEL-NEXT: v_accvgpr_read_b32 v58, a122
+; GISEL-NEXT: v_accvgpr_read_b32 v59, a123
+; GISEL-NEXT: v_accvgpr_read_b32 v60, a124
+; GISEL-NEXT: v_accvgpr_read_b32 v61, a125
+; GISEL-NEXT: v_accvgpr_read_b32 v62, a126
+; GISEL-NEXT: v_accvgpr_read_b32 v63, a127
+; GISEL-NEXT: ds_write_b128 v98, v[64:67]
+; GISEL-NEXT: ds_write_b128 v98, v[68:71] offset:16
+; GISEL-NEXT: ds_write_b128 v98, v[72:75] offset:32
+; GISEL-NEXT: ds_write_b128 v98, v[76:79] offset:48
+; GISEL-NEXT: ds_write_b128 v98, v[80:83] offset:64
+; GISEL-NEXT: ds_write_b128 v98, v[84:87] offset:80
+; GISEL-NEXT: ds_write_b128 v98, v[88:91] offset:96
+; GISEL-NEXT: ds_write_b128 v98, v[92:95] offset:112
+; GISEL-NEXT: v_accvgpr_read_b32 v64, a64
+; GISEL-NEXT: v_accvgpr_read_b32 v65, a65
+; GISEL-NEXT: v_accvgpr_read_b32 v66, a66
+; GISEL-NEXT: v_accvgpr_read_b32 v67, a67
+; GISEL-NEXT: v_accvgpr_read_b32 v68, a68
+; GISEL-NEXT: v_accvgpr_read_b32 v69, a69
+; GISEL-NEXT: v_accvgpr_read_b32 v70, a70
+; GISEL-NEXT: v_accvgpr_read_b32 v71, a71
+; GISEL-NEXT: v_accvgpr_read_b32 v72, a72
+; GISEL-NEXT: v_accvgpr_read_b32 v73, a73
+; GISEL-NEXT: v_accvgpr_read_b32 v74, a74
+; GISEL-NEXT: v_accvgpr_read_b32 v75, a75
+; GISEL-NEXT: v_accvgpr_read_b32 v76, a76
+; GISEL-NEXT: v_accvgpr_read_b32 v77, a77
+; GISEL-NEXT: v_accvgpr_read_b32 v78, a78
+; GISEL-NEXT: v_accvgpr_read_b32 v79, a79
+; GISEL-NEXT: v_accvgpr_read_b32 v80, a80
+; GISEL-NEXT: v_accvgpr_read_b32 v81, a81
+; GISEL-NEXT: v_accvgpr_read_b32 v82, a82
+; GISEL-NEXT: v_accvgpr_read_b32 v83, a83
+; GISEL-NEXT: v_accvgpr_read_b32 v84, a84
+; GISEL-NEXT: v_accvgpr_read_b32 v85, a85
+; GISEL-NEXT: v_accvgpr_read_b32 v86, a86
+; GISEL-NEXT: v_accvgpr_read_b32 v87, a87
+; GISEL-NEXT: v_accvgpr_read_b32 v88, a88
+; GISEL-NEXT: v_accvgpr_read_b32 v89, a89
+; GISEL-NEXT: v_accvgpr_read_b32 v90, a90
+; GISEL-NEXT: v_accvgpr_read_b32 v91, a91
+; GISEL-NEXT: v_accvgpr_read_b32 v92, a92
+; GISEL-NEXT: v_accvgpr_read_b32 v93, a93
+; GISEL-NEXT: v_accvgpr_read_b32 v94, a94
+; GISEL-NEXT: v_accvgpr_read_b32 v95, a95
+; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:16496
; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
@@ -1135,94 +1199,30 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
-; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
-; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
-; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
-; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
-; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
-; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
-; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: ds_write_b128 v96, v[32:35] offset:8192
+; GISEL-NEXT: ds_write_b128 v96, v[36:39] offset:8208
+; GISEL-NEXT: ds_write_b128 v96, v[40:43] offset:8224
+; GISEL-NEXT: ds_write_b128 v96, v[44:47] offset:8240
+; GISEL-NEXT: ds_write_b128 v96, v[48:51] offset:8256
+; GISEL-NEXT: ds_write_b128 v96, v[52:55] offset:8272
+; GISEL-NEXT: ds_write_b128 v96, v[56:59] offset:8288
+; GISEL-NEXT: ds_write_b128 v96, v[60:63] offset:8304
+; GISEL-NEXT: ds_write_b128 v96, v[64:67] offset:24576
+; GISEL-NEXT: ds_write_b128 v96, v[68:71] offset:24592
+; GISEL-NEXT: ds_write_b128 v96, v[72:75] offset:24608
+; GISEL-NEXT: ds_write_b128 v96, v[76:79] offset:24624
+; GISEL-NEXT: ds_write_b128 v96, v[80:83] offset:24640
+; GISEL-NEXT: ds_write_b128 v96, v[84:87] offset:24656
+; GISEL-NEXT: ds_write_b128 v96, v[88:91] offset:24672
+; GISEL-NEXT: ds_write_b128 v96, v[92:95] offset:24688
+; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:32880
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
; GISEL-NEXT: s_endpgm
@@ -1236,23 +1236,23 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v1, 2.0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_add_u32_e32 v3, s0, v0
-; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v3 offset:112
-; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v3 offset:96
-; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v3 offset:80
-; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v3 offset:64
-; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v3
-; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v3 offset:16
-; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v3 offset:32
-; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v3 offset:48
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v3
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v3 offset:48
; EXACTCUTOFF-NEXT: v_add_u32_e32 v4, 0x6000, v3
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v3 offset:8304
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v3 offset:8288
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v3 offset:8272
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v3 offset:8256
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v3 offset:8240
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v3 offset:8224
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v3 offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v3 offset:8192
+; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v3 offset:8304
+; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v3 offset:8288
+; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v3 offset:8272
+; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v3 offset:8256
+; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v3 offset:8240
+; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v3 offset:8224
+; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v3 offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v3 offset:8192
; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v3 offset:24688
; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v3 offset:24672
; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v3 offset:24656
@@ -1278,33 +1278,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v4 offset:57376
; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v4 offset:57392
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(14)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s1, v0
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(8)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
; EXACTCUTOFF-NEXT: s_nop 11
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35]
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3]
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s1
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:8288
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:8304
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:8256
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:8272
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:8224
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:8240
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:8208
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448
@@ -1979,49 +1979,65 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
; SDAG: ; %bb.0: ; %entry
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; SDAG-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
+; SDAG-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; SDAG-NEXT: v_mov_b32_e32 v7, 0x32a5705f
+; SDAG-NEXT: v_mov_b32_e32 v6, 0x32a5705f
; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mul_f32_e32 v4, s0, v3
-; SDAG-NEXT: v_rndne_f32_e32 v5, v4
-; SDAG-NEXT: v_sub_f32_e32 v6, v4, v5
-; SDAG-NEXT: v_fma_f32 v4, s0, v3, -v4
-; SDAG-NEXT: v_fmac_f32_e32 v4, s0, v7
-; SDAG-NEXT: v_add_f32_e32 v4, v6, v4
-; SDAG-NEXT: v_exp_f32_e32 v4, v4
-; SDAG-NEXT: v_cvt_i32_f32_e32 v5, v5
+; SDAG-NEXT: v_mul_f32_e32 v3, s0, v2
+; SDAG-NEXT: v_rndne_f32_e32 v4, v3
+; SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
+; SDAG-NEXT: v_fma_f32 v3, s0, v2, -v3
+; SDAG-NEXT: v_fmac_f32_e32 v3, s0, v6
+; SDAG-NEXT: v_add_f32_e32 v3, v5, v3
+; SDAG-NEXT: v_exp_f32_e32 v3, v3
+; SDAG-NEXT: v_cvt_i32_f32_e32 v4, v4
; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
; SDAG-NEXT: v_add_u32_e32 v1, s6, v0
-; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112
-; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96
-; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80
-; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64
-; SDAG-NEXT: ds_read_b128 a[32:35], v1
-; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16
-; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32
-; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48
-; SDAG-NEXT: v_mov_b32_e32 v9, 1.0
-; SDAG-NEXT: v_ldexp_f32 v4, v4, v5
-; SDAG-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
-; SDAG-NEXT: v_mul_f32_e32 v10, s1, v3
-; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x42b17218
-; SDAG-NEXT: v_rndne_f32_e32 v11, v10
-; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[0:3], v1
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
+; SDAG-NEXT: v_ldexp_f32 v3, v3, v4
+; SDAG-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v4
+; SDAG-NEXT: v_mov_b32_e32 v7, 0x42b17218
+; SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v7
; SDAG-NEXT: v_mov_b32_e32 v8, 0x7f800000
-; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11
-; SDAG-NEXT: v_fma_f32 v10, s1, v3, -v10
-; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; SDAG-NEXT: v_fmac_f32_e32 v10, s1, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc
+; SDAG-NEXT: v_add_u32_e32 v0, s7, v0
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v4
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31]
+; SDAG-NEXT: s_load_dword s8, s[4:5], 0x54
+; SDAG-NEXT: ; kill: killed $sgpr4_sgpr5
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[0:3]
+; SDAG-NEXT: v_mul_f32_e32 v0, s1, v2
+; SDAG-NEXT: v_rndne_f32_e32 v3, v0
+; SDAG-NEXT: v_sub_f32_e32 v9, v0, v3
+; SDAG-NEXT: v_fma_f32 v0, s1, v2, -v0
+; SDAG-NEXT: v_fmac_f32_e32 v0, s1, v6
+; SDAG-NEXT: v_add_f32_e32 v0, v9, v0
+; SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; SDAG-NEXT: s_waitcnt lgkmcnt(1)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
-; SDAG-NEXT: v_add_f32_e32 v4, v12, v10
-; SDAG-NEXT: v_exp_f32_e32 v4, v4
-; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11
; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:8288
; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:8272
; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:8256
@@ -2029,104 +2045,32 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:8224
; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:8208
; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; SDAG-NEXT: v_ldexp_f32 v4, v4, v10
-; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
-; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6
-; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; SDAG-NEXT: v_mul_f32_e32 v10, s2, v3
-; SDAG-NEXT: v_rndne_f32_e32 v11, v10
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
-; SDAG-NEXT: v_fma_f32 v4, s2, v3, -v10
-; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11
-; SDAG-NEXT: v_fmac_f32_e32 v4, s2, v7
-; SDAG-NEXT: v_add_f32_e32 v4, v12, v4
-; SDAG-NEXT: v_exp_f32_e32 v4, v4
-; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11
-; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608
-; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; SDAG-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:49264
-; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:49248
-; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:49232
-; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:49216
-; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:49200
-; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:49184
-; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:49168
-; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:49152
-; SDAG-NEXT: v_ldexp_f32 v1, v4, v10
-; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
-; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
-; SDAG-NEXT: v_mul_f32_e32 v4, s3, v3
-; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; SDAG-NEXT: v_rndne_f32_e32 v10, v4
-; SDAG-NEXT: s_load_dword s8, s[4:5], 0x54
+; SDAG-NEXT: v_ldexp_f32 v0, v0, v3
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; SDAG-NEXT: v_mul_f32_e32 v3, s2, v2
+; SDAG-NEXT: v_rndne_f32_e32 v10, v3
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95]
-; SDAG-NEXT: v_sub_f32_e32 v1, v4, v10
-; SDAG-NEXT: v_fma_f32 v4, s3, v3, -v4
-; SDAG-NEXT: v_fmac_f32_e32 v4, s3, v7
-; SDAG-NEXT: v_add_f32_e32 v1, v1, v4
-; SDAG-NEXT: v_exp_f32_e32 v1, v1
-; SDAG-NEXT: v_cvt_i32_f32_e32 v4, v10
-; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
-; SDAG-NEXT: ds_read_b128 a[156:159], v2 offset:57456
-; SDAG-NEXT: ds_read_b128 a[152:155], v2 offset:57440
-; SDAG-NEXT: v_ldexp_f32 v1, v1, v4
-; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
-; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; SDAG-NEXT: v_mul_f32_e32 v4, s8, v3
-; SDAG-NEXT: v_fma_f32 v3, s8, v3, -v4
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v1, a[96:127]
-; SDAG-NEXT: v_rndne_f32_e32 v1, v4
-; SDAG-NEXT: v_sub_f32_e32 v10, v4, v1
-; SDAG-NEXT: v_fmac_f32_e32 v3, s8, v7
-; SDAG-NEXT: v_add_f32_e32 v3, v10, v3
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v0, a[0:31]
+; SDAG-NEXT: v_sub_f32_e32 v11, v3, v10
+; SDAG-NEXT: v_fma_f32 v3, s2, v2, -v3
+; SDAG-NEXT: v_fmac_f32_e32 v3, s2, v6
+; SDAG-NEXT: v_add_f32_e32 v3, v11, v3
; SDAG-NEXT: v_exp_f32_e32 v3, v3
-; SDAG-NEXT: v_cvt_i32_f32_e32 v1, v1
-; SDAG-NEXT: ds_read_b128 a[148:151], v2 offset:57424
-; SDAG-NEXT: ds_read_b128 a[144:147], v2 offset:57408
-; SDAG-NEXT: ds_read_b128 a[128:131], v2 offset:57344
-; SDAG-NEXT: ds_read_b128 a[132:135], v2 offset:57360
-; SDAG-NEXT: ds_read_b128 a[136:139], v2 offset:57376
-; SDAG-NEXT: ds_read_b128 a[140:143], v2 offset:57392
-; SDAG-NEXT: v_ldexp_f32 v1, v3, v1
-; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5
-; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
-; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; SDAG-NEXT: v_add_u32_e32 v0, s7, v0
-; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; SDAG-NEXT: s_waitcnt lgkmcnt(1)
-; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
-; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v10
; SDAG-NEXT: v_mov_b32_e32 v0, s7
-; SDAG-NEXT: ; kill: killed $sgpr4_sgpr5
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v4
+; SDAG-NEXT: v_add_u32_e32 v9, 0x6000, v1
+; SDAG-NEXT: v_ldexp_f32 v3, v3, v10
+; SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v4
; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 4
; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
@@ -2135,216 +2079,202 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:24672
-; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:24688
-; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:24640
-; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:24656
-; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:24608
-; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:24624
-; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:24576
-; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:24592
-; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:32864
-; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:32880
-; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:32832
-; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:32848
-; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32800
-; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:32816
-; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:32768
-; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:32784
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
-; GISEL: ; %bb.0: ; %entry
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:24688
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:24672
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:24656
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:24640
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:24624
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:24608
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:24592
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:24576
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31]
+; SDAG-NEXT: v_mul_f32_e32 v3, s3, v2
+; SDAG-NEXT: v_rndne_f32_e32 v10, v3
+; SDAG-NEXT: v_sub_f32_e32 v11, v3, v10
+; SDAG-NEXT: v_fma_f32 v3, s3, v2, -v3
+; SDAG-NEXT: v_fmac_f32_e32 v3, s3, v6
+; SDAG-NEXT: v_add_f32_e32 v3, v11, v3
+; SDAG-NEXT: v_exp_f32_e32 v3, v3
+; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v10
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16400
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; SDAG-NEXT: v_ldexp_f32 v1, v3, v10
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v4
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31]
+; SDAG-NEXT: v_mul_f32_e32 v1, s8, v2
+; SDAG-NEXT: v_rndne_f32_e32 v3, v1
+; SDAG-NEXT: v_sub_f32_e32 v10, v1, v3
+; SDAG-NEXT: v_fma_f32 v1, s8, v2, -v1
+; SDAG-NEXT: v_fmac_f32_e32 v1, s8, v6
+; SDAG-NEXT: v_add_f32_e32 v1, v10, v1
+; SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: v_ldexp_f32 v1, v1, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:24592
+; SDAG-NEXT: ds_read_b128 a[28:31], v9 offset:57456
+; SDAG-NEXT: ds_read_b128 a[24:27], v9 offset:57440
+; SDAG-NEXT: ds_read_b128 a[20:23], v9 offset:57424
+; SDAG-NEXT: ds_read_b128 a[16:19], v9 offset:57408
+; SDAG-NEXT: ds_read_b128 a[0:3], v9 offset:57344
+; SDAG-NEXT: ds_read_b128 a[4:7], v9 offset:57360
+; SDAG-NEXT: ds_read_b128 a[8:11], v9 offset:57376
+; SDAG-NEXT: ds_read_b128 a[12:15], v9 offset:57392
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
+; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; GISEL-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
-; GISEL-NEXT: v_mov_b32_e32 v5, 0x32a5705f
-; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mul_f32_e32 v3, s0, v2
-; GISEL-NEXT: v_fma_f32 v4, s0, v2, -v3
-; GISEL-NEXT: v_rndne_f32_e32 v6, v3
-; GISEL-NEXT: v_fmac_f32_e32 v4, s0, v5
-; GISEL-NEXT: v_sub_f32_e32 v3, v3, v6
-; GISEL-NEXT: v_add_f32_e32 v3, v3, v4
-; GISEL-NEXT: v_cvt_i32_f32_e32 v4, v6
-; GISEL-NEXT: v_exp_f32_e32 v3, v3
+; GISEL-NEXT: v_mov_b32_e32 v34, 0x3fb8aa3b
; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v35, 0x32a5705f
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mul_f32_e32 v0, s0, v34
+; GISEL-NEXT: v_fma_f32 v1, s0, v34, -v0
+; GISEL-NEXT: v_rndne_f32_e32 v2, v0
+; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GISEL-NEXT: v_fmac_f32_e32 v1, s0, v35
+; GISEL-NEXT: v_sub_f32_e32 v0, v0, v2
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-NEXT: v_cvt_i32_f32_e32 v1, v2
+; GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v33, s6, v32
+; GISEL-NEXT: v_mov_b32_e32 v36, 0xc2ce8ed0
+; GISEL-NEXT: ds_read_b128 a[0:3], v33
+; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:112
+; GISEL-NEXT: v_ldexp_f32 v0, v0, v1
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v36
+; GISEL-NEXT: v_mov_b32_e32 v37, 0x42b17218
+; GISEL-NEXT: v_mov_b32_e32 v38, 1.0
; GISEL-NEXT: s_load_dword s8, s[4:5], 0x54
; GISEL-NEXT: ; kill: killed $sgpr4_sgpr5
-; GISEL-NEXT: v_add_u32_e32 v0, s6, v32
-; GISEL-NEXT: v_ldexp_f32 v3, v3, v4
-; GISEL-NEXT: v_readfirstlane_b32 s4, v3
-; GISEL-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
-; GISEL-NEXT: ds_read_b128 a[64:67], v0
-; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:16
-; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:32
-; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:48
-; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:64
-; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:80
-; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:96
-; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:112
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v3
-; GISEL-NEXT: v_mov_b32_e32 v4, 0x42b17218
-; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v4
-; GISEL-NEXT: v_mul_f32_e32 v6, s1, v2
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v37
; GISEL-NEXT: s_cselect_b32 s4, 0, s4
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fma_f32 v7, s1, v2, -v6
-; GISEL-NEXT: v_rndne_f32_e32 v8, v6
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s4
-; GISEL-NEXT: v_fmac_f32_e32 v7, s1, v5
-; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8
-; GISEL-NEXT: v_add_f32_e32 v6, v6, v7
-; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
-; GISEL-NEXT: v_mov_b32_e32 v8, s0
-; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192
-; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: v_mul_f32_e32 v39, s1, v34
+; GISEL-NEXT: v_fma_f32 v40, s1, v34, -v39
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v8, a[64:95]
-; GISEL-NEXT: v_exp_f32_e32 v6, v6
-; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224
-; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240
-; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256
-; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272
-; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288
-; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304
-; GISEL-NEXT: v_ldexp_f32 v6, v6, v7
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v3
-; GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v0, a[0:31]
+; GISEL-NEXT: v_rndne_f32_e32 v41, v39
+; GISEL-NEXT: v_fmac_f32_e32 v40, s1, v35
+; GISEL-NEXT: v_sub_f32_e32 v39, v39, v41
+; GISEL-NEXT: v_add_f32_e32 v39, v39, v40
+; GISEL-NEXT: v_cvt_i32_f32_e32 v40, v41
+; GISEL-NEXT: v_exp_f32_e32 v39, v39
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v36
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v4
-; GISEL-NEXT: v_mul_f32_e32 v6, s2, v2
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v37
+; GISEL-NEXT: v_ldexp_f32 v39, v39, v40
+; GISEL-NEXT: v_readfirstlane_b32 s0, v39
; GISEL-NEXT: s_cselect_b32 s0, 0, s0
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fma_f32 v7, s2, v2, -v6
-; GISEL-NEXT: v_rndne_f32_e32 v8, v6
; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_fmac_f32_e32 v7, s2, v5
-; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8
-; GISEL-NEXT: v_add_f32_e32 v6, v6, v7
-; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
-; GISEL-NEXT: v_mov_b32_e32 v8, s0
-; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576
-; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592
-; GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v8, a[0:31]
-; GISEL-NEXT: v_exp_f32_e32 v6, v6
-; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608
-; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624
-; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640
-; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656
-; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672
-; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688
-; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
-; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152
-; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168
-; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184
-; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200
-; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216
-; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232
-; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248
-; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264
-; GISEL-NEXT: v_ldexp_f32 v0, v6, v7
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v3
-; GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v4
-; GISEL-NEXT: v_mul_f32_e32 v0, s3, v2
-; GISEL-NEXT: s_cselect_b32 s0, 0, s0
-; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fma_f32 v6, s3, v2, -v0
-; GISEL-NEXT: v_rndne_f32_e32 v7, v0
-; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_fmac_f32_e32 v6, s3, v5
-; GISEL-NEXT: v_sub_f32_e32 v0, v0, v7
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v6
-; GISEL-NEXT: v_cvt_i32_f32_e32 v6, v7
-; GISEL-NEXT: v_mov_b32_e32 v7, s0
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v3
-; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: s_waitcnt lgkmcnt(8)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v7, a[32:63]
-; GISEL-NEXT: v_exp_f32_e32 v0, v0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v4
-; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
-; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
-; GISEL-NEXT: v_ldexp_f32 v0, v0, v6
-; GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GISEL-NEXT: v_mul_f32_e32 v0, s8, v2
-; GISEL-NEXT: v_fma_f32 v2, s8, v2, -v0
-; GISEL-NEXT: s_cselect_b32 s0, 0, s0
-; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_fmac_f32_e32 v2, s8, v5
-; GISEL-NEXT: v_rndne_f32_e32 v5, v0
-; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_sub_f32_e32 v0, v0, v5
-; GISEL-NEXT: v_add_f32_e32 v0, v0, v2
-; GISEL-NEXT: v_cvt_i32_f32_e32 v2, v5
-; GISEL-NEXT: v_mov_b32_e32 v5, s0
-; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
-; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
-; GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v5, a[96:127]
-; GISEL-NEXT: v_exp_f32_e32 v0, v0
-; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
-; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
-; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
-; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
-; GISEL-NEXT: v_ldexp_f32 v0, v0, v2
-; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v3
-; GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v4
-; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: v_mov_b32_e32 v39, s0
+; GISEL-NEXT: v_add_u32_e32 v32, s7, v32
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v36
; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
-; GISEL-NEXT: v_mov_b32_e32 v34, s0
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v37
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_read_b128 a[0:3], v33 offset:8192
+; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:8208
+; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:8224
+; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:8240
+; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:8256
+; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:8272
+; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:8288
+; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:8304
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
-; GISEL-NEXT: v_add_u32_e32 v32, s7, v32
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v39, a[0:31]
; GISEL-NEXT: ds_write_b128 v32, v[0:3]
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
@@ -2353,6 +2283,22 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
+; GISEL-NEXT: v_mul_f32_e32 v32, s2, v34
+; GISEL-NEXT: v_fma_f32 v39, s2, v34, -v32
+; GISEL-NEXT: v_rndne_f32_e32 v40, v32
+; GISEL-NEXT: v_fmac_f32_e32 v39, s2, v35
+; GISEL-NEXT: v_sub_f32_e32 v32, v32, v40
+; GISEL-NEXT: v_add_f32_e32 v32, v32, v39
+; GISEL-NEXT: v_cvt_i32_f32_e32 v39, v40
+; GISEL-NEXT: v_exp_f32_e32 v32, v32
+; GISEL-NEXT: v_add_u32_e32 v40, 0xc000, v33
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: v_ldexp_f32 v32, v32, v39
+; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
@@ -2385,6 +2331,29 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_read_b128 a[0:3], v33 offset:24576
+; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:24592
+; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:24608
+; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:24624
+; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:24640
+; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:24656
+; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:24672
+; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:24688
+; GISEL-NEXT: v_readfirstlane_b32 s0, v32
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_mov_b32_e32 v32, s0
+; GISEL-NEXT: v_mul_f32_e32 v39, s3, v34
+; GISEL-NEXT: v_fma_f32 v41, s3, v34, -v39
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v32, a[0:31]
+; GISEL-NEXT: v_rndne_f32_e32 v42, v39
+; GISEL-NEXT: v_fmac_f32_e32 v41, s3, v35
+; GISEL-NEXT: v_sub_f32_e32 v39, v39, v42
+; GISEL-NEXT: v_add_f32_e32 v39, v39, v41
+; GISEL-NEXT: v_cvt_i32_f32_e32 v41, v42
+; GISEL-NEXT: v_exp_f32_e32 v39, v39
; GISEL-NEXT: v_mov_b32_e32 v32, s7
; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
@@ -2394,128 +2363,171 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v36
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v37
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_read_b128 a[0:3], v33 offset:49152
+; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:49168
+; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:49184
+; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:49200
+; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:49216
+; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:49232
+; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:49248
+; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:49264
+; GISEL-NEXT: v_ldexp_f32 v33, v39, v41
+; GISEL-NEXT: v_readfirstlane_b32 s0, v33
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_mov_b32_e32 v33, s0
; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
+; GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v33, a[0:31]
+; GISEL-NEXT: v_mul_f32_e32 v33, s8, v34
+; GISEL-NEXT: v_fma_f32 v34, s8, v34, -v33
+; GISEL-NEXT: v_fmac_f32_e32 v34, s8, v35
+; GISEL-NEXT: v_rndne_f32_e32 v35, v33
+; GISEL-NEXT: v_sub_f32_e32 v33, v33, v35
+; GISEL-NEXT: v_add_f32_e32 v33, v33, v34
+; GISEL-NEXT: v_cvt_i32_f32_e32 v34, v35
+; GISEL-NEXT: v_exp_f32_e32 v33, v33
; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: v_ldexp_f32 v33, v33, v34
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v36
+; GISEL-NEXT: v_readfirstlane_b32 s0, v33
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v37
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_read_b128 a[0:3], v40 offset:32768
+; GISEL-NEXT: ds_read_b128 a[4:7], v40 offset:32784
+; GISEL-NEXT: ds_read_b128 a[8:11], v40 offset:32800
+; GISEL-NEXT: ds_read_b128 a[12:15], v40 offset:32816
+; GISEL-NEXT: ds_read_b128 a[16:19], v40 offset:32832
+; GISEL-NEXT: ds_read_b128 a[20:23], v40 offset:32848
+; GISEL-NEXT: ds_read_b128 a[24:27], v40 offset:32864
+; GISEL-NEXT: ds_read_b128 a[28:31], v40 offset:32880
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_mov_b32_e32 v33, s0
; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
+; GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v33, a[0:31]
; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
-; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
-; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
-; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
-; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
-; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
-; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
-; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
-; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
-; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
-; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
-; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
-; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
-; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
-; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
-; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
-; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
-; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
-; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
-; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
-; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
-; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
-; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
-; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
-; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
-; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
-; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
-; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
-; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
-; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
-; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
-; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
-; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 12
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
@@ -2529,49 +2541,65 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
; EXACTCUTOFF: ; %bb.0: ; %entry
; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
; EXACTCUTOFF-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v7, 0x32a5705f
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v6, 0x32a5705f
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s0, v3
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v5, v4
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v6, v4, v5
-; EXACTCUTOFF-NEXT: v_fma_f32 v4, s0, v3, -v4
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s0, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v6, v4
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v5, v5
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s0, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v4, v3
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v5, v3, v4
+; EXACTCUTOFF-NEXT: v_fma_f32 v3, s0, v2, -v3
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s0, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v5, v3
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v4, v4
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s6, v0
-; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:112
-; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:96
-; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:80
-; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:64
-; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:16
-; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:32
-; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:48
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v9, 1.0
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v5
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s1, v3
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v6, 0x42b17218
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 1.0
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v4
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v4
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v7, 0x42b17218
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v7
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v8, 0x7f800000
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11
-; EXACTCUTOFF-NEXT: v_fma_f32 v10, s1, v3, -v10
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v10, s1, v7
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s7, v0
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v4
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31]
+; EXACTCUTOFF-NEXT: s_load_dword s8, s[4:5], 0x54
+; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: s_nop 15
+; EXACTCUTOFF-NEXT: s_nop 1
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3]
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v0, s1, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v3, v0
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v9, v0, v3
+; EXACTCUTOFF-NEXT: v_fma_f32 v0, s1, v2, -v0
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v0, s1, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v0, v9, v0
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v0, v0
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v3, v3
; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v10
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11
; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:8288
; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:8272
; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:8256
@@ -2579,104 +2607,32 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:8224
; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:8208
; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v10
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s2, v3
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v0, v0, v3
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v7
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s2, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v3
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
-; EXACTCUTOFF-NEXT: v_fma_f32 v4, s2, v3, -v10
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s2, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v4
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11
-; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; EXACTCUTOFF-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; EXACTCUTOFF-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; EXACTCUTOFF-NEXT: ds_read_b128 a[72:75], v1 offset:24608
-; EXACTCUTOFF-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; EXACTCUTOFF-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[124:127], v1 offset:49264
-; EXACTCUTOFF-NEXT: ds_read_b128 a[120:123], v1 offset:49248
-; EXACTCUTOFF-NEXT: ds_read_b128 a[116:119], v1 offset:49232
-; EXACTCUTOFF-NEXT: ds_read_b128 a[112:115], v1 offset:49216
-; EXACTCUTOFF-NEXT: ds_read_b128 a[108:111], v1 offset:49200
-; EXACTCUTOFF-NEXT: ds_read_b128 a[104:107], v1 offset:49184
-; EXACTCUTOFF-NEXT: ds_read_b128 a[100:103], v1 offset:49168
-; EXACTCUTOFF-NEXT: ds_read_b128 a[96:99], v1 offset:49152
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v4, v10
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s3, v3
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v4
-; EXACTCUTOFF-NEXT: s_load_dword s8, s[4:5], 0x54
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95]
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v1, v4, v10
-; EXACTCUTOFF-NEXT: v_fma_f32 v4, s3, v3, -v4
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s3, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v1, v1, v4
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v1, v1
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v4, v10
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
-; EXACTCUTOFF-NEXT: ds_read_b128 a[156:159], v2 offset:57456
-; EXACTCUTOFF-NEXT: ds_read_b128 a[152:155], v2 offset:57440
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v1, v4
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s8, v3
-; EXACTCUTOFF-NEXT: v_fma_f32 v3, s8, v3, -v4
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v1, a[96:127]
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v1, v4
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v10, v4, v1
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s8, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v10, v3
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v0, a[0:31]
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v11, v3, v10
+; EXACTCUTOFF-NEXT: v_fma_f32 v3, s2, v2, -v3
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s2, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v11, v3
; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v1, v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[148:151], v2 offset:57424
-; EXACTCUTOFF-NEXT: ds_read_b128 a[144:147], v2 offset:57408
-; EXACTCUTOFF-NEXT: ds_read_b128 a[128:131], v2 offset:57344
-; EXACTCUTOFF-NEXT: ds_read_b128 a[132:135], v2 offset:57360
-; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v2 offset:57376
-; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v2 offset:57392
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v3, v1
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s7, v0
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35]
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v10
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s7
-; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v4
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v9, 0x6000, v1
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v10
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v7
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v4
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: s_nop 4
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
@@ -2685,30 +2641,94 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[120:123] offset:24672
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[124:127] offset:24688
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[112:115] offset:24640
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[116:119] offset:24656
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[104:107] offset:24608
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[108:111] offset:24624
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[96:99] offset:24576
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[100:103] offset:24592
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[152:155] offset:32864
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[156:159] offset:32880
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[144:147] offset:32832
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[148:151] offset:32848
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[136:139] offset:32800
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[140:143] offset:32816
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[128:131] offset:32768
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[132:135] offset:32784
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:24688
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:24672
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:24656
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:24640
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:24624
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:24608
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:24592
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:24576
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31]
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s3, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v3
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v11, v3, v10
+; EXACTCUTOFF-NEXT: v_fma_f32 v3, s3, v2, -v3
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s3, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v11, v3
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v10
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: s_nop 10
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:16480
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:16496
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:16448
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:16464
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:16416
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:16432
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:16384
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16400
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v3, v10
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v7
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v4
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31]
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v1, s8, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v3, v1
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v10, v1, v3
+; EXACTCUTOFF-NEXT: v_fma_f32 v1, s8, v2, -v1
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v1, s8, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v1, v10, v1
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v1, v1
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v2, v3
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v1, v2
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v7
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; EXACTCUTOFF-NEXT: s_nop 6
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:24592
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v9 offset:57456
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v9 offset:57440
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v9 offset:57424
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v9 offset:57408
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v9 offset:57344
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v9 offset:57360
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v9 offset:57376
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v9 offset:57392
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31]
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: s_nop 15
+; EXACTCUTOFF-NEXT: s_nop 2
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:32864
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:32880
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:32832
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:32848
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32800
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:32816
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:32768
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:32784
; EXACTCUTOFF-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
index 8b928bc9defa5..c8c71709ad803 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
@@ -790,12 +790,6 @@ body: |
; GFX908-NEXT: [[DEF20:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX908-NEXT: [[DEF21:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX908-NEXT: [[DEF22:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX908-NEXT: [[DEF23:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX908-NEXT: [[DEF24:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX908-NEXT: [[DEF25:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX908-NEXT: [[DEF26:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX908-NEXT: [[DEF27:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX908-NEXT: [[DEF28:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX908-NEXT: [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: [[V_CVT_I32_F64_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: [[V_CVT_I32_F64_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
@@ -822,6 +816,12 @@ body: |
; GFX908-NEXT: [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
+ ; GFX908-NEXT: [[DEF23:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX908-NEXT: [[DEF24:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX908-NEXT: [[DEF25:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX908-NEXT: [[DEF26:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX908-NEXT: [[DEF27:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX908-NEXT: [[DEF28:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX908-NEXT: [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
; GFX908-NEXT: [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
; GFX908-NEXT: [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
@@ -2268,6 +2268,8 @@ body: |
; GFX90A-NEXT: [[DEF46:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[DEF47:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[DEF48:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+ ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
+ ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
; GFX90A-NEXT: [[DEF49:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[DEF50:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[DEF51:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
@@ -2465,8 +2467,6 @@ body: |
; GFX90A-NEXT: [[DEF243:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[DEF244:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[DEF245:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
- ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
- ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.1:
; GFX90A-NEXT: [[DEF246:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
index fc1c2d7d16d21..f703874d174c3 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg)
; GFX908-LABEL: test_mfma_f32_32x32x1f32_vgpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-NEXT: v_mov_b32_e32 v29, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -21,13 +21,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mov_b32_e32 v2, s18
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, s22
+; GFX908-NEXT: v_mov_b32_e32 v1, s23
; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s21
-; GFX908-NEXT: v_mov_b32_e32 v1, s22
-; GFX908-NEXT: v_mov_b32_e32 v2, s23
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
; GFX908-NEXT: v_mov_b32_e32 v0, s24
; GFX908-NEXT: v_mov_b32_e32 v1, s25
; GFX908-NEXT: v_mov_b32_e32 v2, s26
@@ -68,7 +66,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mov_b32_e32 v0, s10
; GFX908-NEXT: v_mov_b32_e32 v1, s11
; GFX908-NEXT: v_mov_b32_e32 v2, s12
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
+; GFX908-NEXT: v_mov_b32_e32 v5, s21
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
; GFX908-NEXT: v_accvgpr_write_b32 a26, v0
; GFX908-NEXT: v_accvgpr_write_b32 a27, v1
@@ -77,7 +76,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mov_b32_e32 v1, s14
; GFX908-NEXT: v_mov_b32_e32 v2, s15
; GFX908-NEXT: v_mov_b32_e32 v3, 1.0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
; GFX908-NEXT: v_accvgpr_write_b32 a29, v0
; GFX908-NEXT: v_accvgpr_write_b32 a30, v1
; GFX908-NEXT: v_accvgpr_write_b32 a31, v2
@@ -90,57 +90,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a19
+; GFX908-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; GFX908-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; GFX908-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; GFX908-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; GFX908-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; GFX908-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; GFX908-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; GFX908-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
@@ -153,7 +138,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg)
; GFX908-LABEL: test_mfma_f32_32x32x1f32_agpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-NEXT: v_mov_b32_e32 v29, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -163,13 +148,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mov_b32_e32 v2, s18
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, s22
+; GFX908-NEXT: v_mov_b32_e32 v1, s23
; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s21
-; GFX908-NEXT: v_mov_b32_e32 v1, s22
-; GFX908-NEXT: v_mov_b32_e32 v2, s23
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
; GFX908-NEXT: v_mov_b32_e32 v0, s24
; GFX908-NEXT: v_mov_b32_e32 v1, s25
; GFX908-NEXT: v_mov_b32_e32 v2, s26
@@ -210,7 +193,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mov_b32_e32 v0, s10
; GFX908-NEXT: v_mov_b32_e32 v1, s11
; GFX908-NEXT: v_mov_b32_e32 v2, s12
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
+; GFX908-NEXT: v_mov_b32_e32 v5, s21
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
; GFX908-NEXT: v_accvgpr_write_b32 a26, v0
; GFX908-NEXT: v_accvgpr_write_b32 a27, v1
@@ -219,7 +203,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_mov_b32_e32 v1, s14
; GFX908-NEXT: v_mov_b32_e32 v2, s15
; GFX908-NEXT: v_mov_b32_e32 v3, 1.0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
; GFX908-NEXT: v_accvgpr_write_b32 a29, v0
; GFX908-NEXT: v_accvgpr_write_b32 a30, v1
; GFX908-NEXT: v_accvgpr_write_b32 a31, v2
@@ -232,57 +217,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a19
+; GFX908-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112
+; GFX908-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64
+; GFX908-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80
+; GFX908-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32
+; GFX908-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48
+; GFX908-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35]
+; GFX908-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16
; GFX908-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
@@ -351,43 +321,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr(ptr
; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a19
; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112
+; GFX908-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64
+; GFX908-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80
+; GFX908-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32
+; GFX908-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48
+; GFX908-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1]
+; GFX908-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16
; GFX908-NEXT: s_endpgm
bb:
%acc = call i32 asm sideeffect "; def $0", "={a0}"()
@@ -458,43 +427,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr add
; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a19
; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112
+; GFX908-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64
+; GFX908-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80
+; GFX908-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32
+; GFX908-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48
+; GFX908-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1]
+; GFX908-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16
; GFX908-NEXT: s_endpgm
bb:
call void asm sideeffect "; use $0", "{a[100:131]}"(<32 x float> poison)
@@ -564,43 +532,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addr
; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a19
; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112
+; GFX908-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64
+; GFX908-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80
+; GFX908-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32
+; GFX908-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48
+; GFX908-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1]
+; GFX908-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16
; GFX908-NEXT: s_endpgm
bb:
%acc = call i32 asm sideeffect "; def $0", "={v0}"()
@@ -693,43 +660,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a19
; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v40, v[4:7], s[34:35] offset:112
-; GFX908-NEXT: global_store_dwordx4 v40, v[8:11], s[34:35] offset:64
-; GFX908-NEXT: global_store_dwordx4 v40, v[12:15], s[34:35] offset:80
-; GFX908-NEXT: global_store_dwordx4 v40, v[16:19], s[34:35] offset:32
-; GFX908-NEXT: global_store_dwordx4 v40, v[20:23], s[34:35] offset:48
-; GFX908-NEXT: global_store_dwordx4 v40, v[24:27], s[34:35]
-; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a4
+; GFX908-NEXT: global_store_dwordx4 v40, v[1:4], s[34:35] offset:112
+; GFX908-NEXT: global_store_dwordx4 v40, v[5:8], s[34:35] offset:64
+; GFX908-NEXT: global_store_dwordx4 v40, v[9:12], s[34:35] offset:80
+; GFX908-NEXT: global_store_dwordx4 v40, v[13:16], s[34:35] offset:32
+; GFX908-NEXT: global_store_dwordx4 v40, v[17:20], s[34:35] offset:48
+; GFX908-NEXT: global_store_dwordx4 v40, v[21:24], s[34:35]
+; GFX908-NEXT: global_store_dwordx4 v40, v[25:28], s[34:35] offset:16
; GFX908-NEXT: s_endpgm
bb:
call void @foo()
@@ -756,8 +722,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(
; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX908-NEXT: s_load_dword s8, s[4:5], 0x2c
; GFX908-NEXT: v_mov_b32_e32 v6, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v7, 0
; GFX908-NEXT: s_addc_u32 s53, s53, 0
+; GFX908-NEXT: v_mov_b32_e32 v32, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dwordx16 s[36:51], s[6:7], 0x0
; GFX908-NEXT: s_load_dwordx16 s[16:31], s[6:7], 0x40
@@ -837,57 +803,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(
; GFX908-NEXT: v_accvgpr_read_b32 v5, a26
; GFX908-NEXT: v_accvgpr_read_b32 v4, a25
; GFX908-NEXT: v_accvgpr_read_b32 v3, a24
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a28
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:112
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a16
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:64
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a20
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:80
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a8
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:32
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a12
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:48
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a4
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v31, a7
+; GFX908-NEXT: global_store_dwordx4 v32, v[3:6], s[6:7] offset:96
+; GFX908-NEXT: v_accvgpr_read_b32 v30, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v29, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a4
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
+; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[6:7] offset:112
+; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[6:7] offset:64
+; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[6:7] offset:80
+; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[6:7] offset:32
+; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[6:7] offset:48
+; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[6:7]
+; GFX908-NEXT: global_store_dwordx4 v32, v[28:31], s[6:7] offset:16
; GFX908-NEXT: s_cbranch_scc1 .LBB6_2
; GFX908-NEXT: ; %bb.1: ; %bb2
; GFX908-NEXT: s_add_u32 s8, s4, 48
@@ -979,43 +930,42 @@ define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0
; GFX908-NEXT: v_accvgpr_read_b32 v4, a26
; GFX908-NEXT: v_accvgpr_read_b32 v3, a25
; GFX908-NEXT: v_accvgpr_read_b32 v2, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a19
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a4
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v30, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v29, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a4
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off offset:112
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:64
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off offset:80
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off offset:32
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off offset:48
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off offset:16
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
bb:
@@ -1080,43 +1030,42 @@ define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v4, a26
; GFX908-NEXT: v_accvgpr_read_b32 v3, a25
; GFX908-NEXT: v_accvgpr_read_b32 v2, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a19
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a4
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v26, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v30, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a12
+; GFX908-NEXT: v_accvgpr_read_b32 v25, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v29, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v28, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v27, a4
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off offset:112
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:64
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off offset:80
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off offset:32
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off offset:48
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off offset:16
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 07d04ea789a81..7f787568b5a0f 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -19,6 +19,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY908-LABEL: test_mfma_f32_32x32x1f32:
; GREEDY908: ; %bb.0: ; %bb
; GREEDY908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GREEDY908-NEXT: v_mov_b32_e32 v32, 0
; GREEDY908-NEXT: s_waitcnt lgkmcnt(0)
; GREEDY908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
; GREEDY908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
@@ -89,71 +90,70 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v1
; GREEDY908-NEXT: v_accvgpr_write_b32 a31, v2
; GREEDY908-NEXT: v_mov_b32_e32 v0, 2.0
-; GREEDY908-NEXT: v_mov_b32_e32 v4, 0
-; GREEDY908-NEXT: s_nop 0
+; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v3, v0, a[0:31]
; GREEDY908-NEXT: s_nop 15
; GREEDY908-NEXT: s_nop 1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a32
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a33
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a33
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a34
; GREEDY908-NEXT: v_accvgpr_write_b32 a2, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a4, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a35
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a36
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a36
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a37
; GREEDY908-NEXT: v_accvgpr_write_b32 a5, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a7, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a38
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a39
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a39
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a40
; GREEDY908-NEXT: v_accvgpr_write_b32 a8, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a10, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a41
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a42
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a42
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a43
; GREEDY908-NEXT: v_accvgpr_write_b32 a11, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a13, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a44
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a45
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a45
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a46
; GREEDY908-NEXT: v_accvgpr_write_b32 a14, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a16, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a47
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a48
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a48
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a49
; GREEDY908-NEXT: v_accvgpr_write_b32 a17, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a18, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a18, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a19, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a50
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a51
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a51
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a52
; GREEDY908-NEXT: v_accvgpr_write_b32 a20, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a21, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a21, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a22, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a53
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a54
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a54
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a55
; GREEDY908-NEXT: v_accvgpr_write_b32 a23, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a24, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a24, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a25, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a56
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a57
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a57
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a58
; GREEDY908-NEXT: v_accvgpr_write_b32 a26, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a27, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a27, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a28, v1
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a59
-; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a60
+; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a60
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a61
; GREEDY908-NEXT: v_accvgpr_write_b32 a29, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v33
; GREEDY908-NEXT: v_accvgpr_write_b32 a31, v1
; GREEDY908-NEXT: s_nop 0
; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31]
@@ -163,57 +163,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a26
; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a25
; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a24
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a31
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a30
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a29
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a28
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a19
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a18
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a17
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a16
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a23
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a22
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a21
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a20
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a11
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a10
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a9
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a8
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a15
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a14
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a13
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a12
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a3
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a2
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a1
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a0
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a7
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a6
-; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a5
-; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a4
-; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; GREEDY908-NEXT: v_accvgpr_read_b32 v7, a31
+; GREEDY908-NEXT: v_accvgpr_read_b32 v11, a19
+; GREEDY908-NEXT: v_accvgpr_read_b32 v15, a23
+; GREEDY908-NEXT: v_accvgpr_read_b32 v19, a11
+; GREEDY908-NEXT: v_accvgpr_read_b32 v23, a15
+; GREEDY908-NEXT: v_accvgpr_read_b32 v27, a3
+; GREEDY908-NEXT: v_accvgpr_read_b32 v31, a7
+; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a30
+; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a29
+; GREEDY908-NEXT: v_accvgpr_read_b32 v4, a28
+; GREEDY908-NEXT: v_accvgpr_read_b32 v10, a18
+; GREEDY908-NEXT: v_accvgpr_read_b32 v9, a17
+; GREEDY908-NEXT: v_accvgpr_read_b32 v8, a16
+; GREEDY908-NEXT: v_accvgpr_read_b32 v14, a22
+; GREEDY908-NEXT: v_accvgpr_read_b32 v13, a21
+; GREEDY908-NEXT: v_accvgpr_read_b32 v12, a20
+; GREEDY908-NEXT: v_accvgpr_read_b32 v18, a10
+; GREEDY908-NEXT: v_accvgpr_read_b32 v17, a9
+; GREEDY908-NEXT: v_accvgpr_read_b32 v16, a8
+; GREEDY908-NEXT: v_accvgpr_read_b32 v22, a14
+; GREEDY908-NEXT: v_accvgpr_read_b32 v21, a13
+; GREEDY908-NEXT: v_accvgpr_read_b32 v20, a12
+; GREEDY908-NEXT: v_accvgpr_read_b32 v26, a2
+; GREEDY908-NEXT: v_accvgpr_read_b32 v25, a1
+; GREEDY908-NEXT: v_accvgpr_read_b32 v24, a0
+; GREEDY908-NEXT: v_accvgpr_read_b32 v30, a6
+; GREEDY908-NEXT: v_accvgpr_read_b32 v29, a5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v28, a4
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] offset:96
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:112
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:64
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:80
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:32
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:48
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35]
+; GREEDY908-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:16
; GREEDY908-NEXT: s_endpgm
;
; GREEDY90A-LABEL: test_mfma_f32_32x32x1f32:
@@ -582,48 +567,71 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY908-NEXT: s_waitcnt lgkmcnt(0)
; GREEDY908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GREEDY908-NEXT: s_waitcnt lgkmcnt(0)
-; GREEDY908-NEXT: v_mov_b32_e32 v5, s15
-; GREEDY908-NEXT: v_mov_b32_e32 v2, s14
-; GREEDY908-NEXT: v_mov_b32_e32 v1, s13
-; GREEDY908-NEXT: v_accvgpr_write_b32 a33, v5
-; GREEDY908-NEXT: v_mov_b32_e32 v5, s12
-; GREEDY908-NEXT: v_accvgpr_write_b32 a32, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a31, v1
-; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v5
-; GREEDY908-NEXT: v_mov_b32_e32 v2, s11
-; GREEDY908-NEXT: v_mov_b32_e32 v1, s10
-; GREEDY908-NEXT: v_mov_b32_e32 v5, s9
-; GREEDY908-NEXT: v_accvgpr_write_b32 a29, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a28, v1
-; GREEDY908-NEXT: v_accvgpr_write_b32 a27, v5
-; GREEDY908-NEXT: v_mov_b32_e32 v2, s8
-; GREEDY908-NEXT: v_mov_b32_e32 v1, s7
-; GREEDY908-NEXT: v_mov_b32_e32 v5, s6
-; GREEDY908-NEXT: v_accvgpr_write_b32 a26, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a25, v1
-; GREEDY908-NEXT: v_accvgpr_write_b32 a24, v5
-; GREEDY908-NEXT: v_mov_b32_e32 v2, s5
-; GREEDY908-NEXT: v_mov_b32_e32 v1, s4
-; GREEDY908-NEXT: v_mov_b32_e32 v5, s3
-; GREEDY908-NEXT: v_accvgpr_write_b32 a23, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a22, v1
-; GREEDY908-NEXT: v_accvgpr_write_b32 a21, v5
-; GREEDY908-NEXT: v_mov_b32_e32 v2, s2
-; GREEDY908-NEXT: v_mov_b32_e32 v1, s1
; GREEDY908-NEXT: v_mov_b32_e32 v5, s0
-; GREEDY908-NEXT: v_accvgpr_write_b32 a20, v2
-; GREEDY908-NEXT: v_accvgpr_write_b32 a19, v1
-; GREEDY908-NEXT: v_accvgpr_write_b32 a18, v5
+; GREEDY908-NEXT: v_mov_b32_e32 v1, s1
+; GREEDY908-NEXT: v_mov_b32_e32 v2, s2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a0, v5
+; GREEDY908-NEXT: v_mov_b32_e32 v5, s3
+; GREEDY908-NEXT: v_accvgpr_write_b32 a1, v1
+; GREEDY908-NEXT: v_accvgpr_write_b32 a2, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v5
+; GREEDY908-NEXT: v_mov_b32_e32 v1, s4
+; GREEDY908-NEXT: v_mov_b32_e32 v2, s5
+; GREEDY908-NEXT: v_mov_b32_e32 v5, s6
+; GREEDY908-NEXT: v_accvgpr_write_b32 a4, v1
+; GREEDY908-NEXT: v_accvgpr_write_b32 a5, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v5
+; GREEDY908-NEXT: v_mov_b32_e32 v1, s7
+; GREEDY908-NEXT: v_mov_b32_e32 v2, s8
+; GREEDY908-NEXT: v_mov_b32_e32 v5, s9
+; GREEDY908-NEXT: v_accvgpr_write_b32 a7, v1
+; GREEDY908-NEXT: v_accvgpr_write_b32 a8, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v5
+; GREEDY908-NEXT: v_mov_b32_e32 v1, s10
+; GREEDY908-NEXT: v_mov_b32_e32 v2, s11
+; GREEDY908-NEXT: v_mov_b32_e32 v5, s12
+; GREEDY908-NEXT: v_accvgpr_write_b32 a10, v1
+; GREEDY908-NEXT: v_accvgpr_write_b32 a11, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v5
+; GREEDY908-NEXT: v_mov_b32_e32 v1, s13
+; GREEDY908-NEXT: v_mov_b32_e32 v2, s14
+; GREEDY908-NEXT: v_mov_b32_e32 v5, s15
+; GREEDY908-NEXT: v_accvgpr_write_b32 a13, v1
+; GREEDY908-NEXT: v_accvgpr_write_b32 a14, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v5
; GREEDY908-NEXT: v_mov_b32_e32 v1, 2.0
; GREEDY908-NEXT: s_nop 1
-; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[18:33], v0, v1, a[18:33]
-; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[2:17], v0, v1, a[18:33]
-; GREEDY908-NEXT: s_nop 8
-; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a18
-; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a19
-; GREEDY908-NEXT: s_nop 0
-; GREEDY908-NEXT: v_accvgpr_write_b32 a0, v5
-; GREEDY908-NEXT: v_accvgpr_write_b32 a1, v2
+; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
+; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[16:31], v0, v1, a[0:15]
+; GREEDY908-NEXT: s_nop 9
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a16
+; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a17
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a18
+; GREEDY908-NEXT: v_accvgpr_write_b32 a2, v3
+; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a19
+; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a20
+; GREEDY908-NEXT: v_accvgpr_write_b32 a4, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a5, v3
+; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a21
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a22
+; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a23
+; GREEDY908-NEXT: v_accvgpr_write_b32 a7, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a8, v3
+; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a24
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a25
+; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a26
+; GREEDY908-NEXT: v_accvgpr_write_b32 a10, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a11, v3
+; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v5
+; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a27
+; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a28
+; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a29
+; GREEDY908-NEXT: v_accvgpr_write_b32 a13, v2
+; GREEDY908-NEXT: v_accvgpr_write_b32 a14, v3
+; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v5
; GREEDY908-NEXT: s_nop 0
; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
; GREEDY908-NEXT: s_nop 9
diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
index ea6216ac4ac40..1ca643d90aac0 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
@@ -191,82 +191,81 @@ define amdgpu_kernel void @max_10_vgprs_used_9a() #1 {
define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 {
; GFX908-LABEL: max_32regs_mfma32:
; GFX908: ; %bb.0: ; %bb
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x40400000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x40c00000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x40e00000
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v1
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x40a00000
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v3
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v1
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41000000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41100000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41200000
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41300000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41400000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41500000
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41600000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41700000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41800000
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41880000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41900000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41980000
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41a00000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41a80000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41b00000
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41b80000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41c00000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41c80000
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41d00000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41d80000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41e00000
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x41e80000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x41f00000
-; GFX908-NEXT: v_mov_b32_e32 v3, 0x41f80000
-; GFX908-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x40400000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x40c00000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x40e00000
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x40a00000
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v0
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41000000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41100000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41200000
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41300000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41400000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41500000
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41600000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41700000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41800000
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41880000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41900000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41980000
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41a00000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41a80000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41b00000
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41b80000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41c00000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41c80000
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41d00000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41d80000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41e00000
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x41e80000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x41f00000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x41f80000
+; GFX908-NEXT: v_mov_b32_e32 v3, 1.0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a0
; GFX908-NEXT: v_accvgpr_write_b32 a0, 1.0
; GFX908-NEXT: v_accvgpr_write_b32 a1, 2.0
; GFX908-NEXT: v_accvgpr_write_b32 a3, 4.0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v2
; GFX908-NEXT: v_accvgpr_write_b32 a31, 2.0
; GFX908-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v4, v4, a[0:31]
-; GFX908-NEXT: v_mov_b32_e32 v0, 0
-; GFX908-NEXT: s_nop 13
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v5
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v3, a[0:31]
+; GFX908-NEXT: v_mov_b32_e32 v1, 0
+; GFX908-NEXT: s_nop 15
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v4
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX908-NEXT: global_store_dword v1, v0, s[2:3]
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: max_32regs_mfma32:
@@ -330,7 +329,8 @@ define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 {
; GFX90A-NEXT: v_accvgpr_write_b32 a1, 2.0
; GFX90A-NEXT: v_accvgpr_write_b32 a29, v1
; GFX90A-NEXT: v_mov_b32_e32 v1, 0x41f80000
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_write_b32 a0, 1.0
; GFX90A-NEXT: v_accvgpr_write_b32 a3, 4.0
; GFX90A-NEXT: v_accvgpr_write_b32 a30, v1
@@ -338,12 +338,9 @@ define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 {
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v2, a[0:31]
-; GFX90A-NEXT: s_nop 15
-; GFX90A-NEXT: s_nop 2
-; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_nop 15
+; GFX90A-NEXT: s_nop 1
; GFX90A-NEXT: global_store_dword v0, a0, s[2:3]
; GFX90A-NEXT: s_endpgm
bb:
More information about the llvm-commits
mailing list