[llvm] [AMDGPU] Model 2-pass HWVALU blocking for 8-pass XDL MFMA (PR #184084)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 08:09:54 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/184084
>From f3e457fbed4c14f5fba8de3cb9992ae32c34b847 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Wed, 27 May 2026 10:12:35 -0500
Subject: [PATCH] [AMDGPU] Model 2-pass HWVALU blocking for 8-pass XDL MFMA
Reserve HWVALU for two cycles on Write8PassMAI so the machine
scheduler reflects that VALU cannot co-execute with MFMA during
its first two passes. Also record multi-cycle buffered reservations
in SchedBoundary so structural hazards apply to dual-resource writes.
Add schedule-mfma-valu-after-load.mir and refresh affected tests.
---
llvm/lib/CodeGen/MachineScheduler.cpp | 5 +-
llvm/lib/Target/AMDGPU/SISchedule.td | 6 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 219 +-
.../AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir | 1805 +++++++++--------
.../AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir | 89 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll | 20 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 268 +--
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 1088 +++++-----
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll | 20 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 68 +-
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 68 +-
.../AMDGPU/neighboring-mfma-padding.mir | 22 +-
.../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll | 13 +-
.../AMDGPU/schedule-mfma-valu-after-load.mir | 31 +
.../CodeGen/AMDGPU/schedule-pending-queue.mir | 13 +-
...ck-non-coissue-insts-post-ra-scheduler.mir | 66 +-
llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s | 60 +-
llvm/test/tools/llvm-mca/AMDGPU/gfx950.s | 148 +-
18 files changed, 2073 insertions(+), 1936 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 786fe1d0b8fe3..d35db3dcb8bf0 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -3013,7 +3013,10 @@ void SchedBoundary::bumpNode(SUnit *SU) {
PI = SchedModel->getWriteProcResBegin(SC),
PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
unsigned PIdx = PI->ProcResourceIdx;
- if (SchedModel->getResourceBufferSize(PIdx) == 0) {
+ // Track unbuffered resources, and buffered resources held for multiple
+ // cycles.
+ if (SchedModel->getResourceBufferSize(PIdx) == 0 ||
+ PI->ReleaseAtCycle > PI->AcquireAtCycle) {
if (SchedModel && SchedModel->enableIntervals()) {
unsigned ReservedUntil, InstanceIdx;
diff --git a/llvm/lib/Target/AMDGPU/SISchedule.td b/llvm/lib/Target/AMDGPU/SISchedule.td
index 295a7dbe5cfd7..bfb78e2172e88 100644
--- a/llvm/lib/Target/AMDGPU/SISchedule.td
+++ b/llvm/lib/Target/AMDGPU/SISchedule.td
@@ -195,8 +195,10 @@ multiclass SICommonWriteRes {
def : HWWriteRes<Write2PassMAI, [HWXDL], 2>;
let ReleaseAtCycles = [4] in
def : HWWriteRes<Write4PassMAI, [HWXDL], 4>;
- let ReleaseAtCycles = [8] in
- def : HWWriteRes<Write8PassMAI, [HWXDL], 8>;
+ // 8-pass XDL MFMA blocks HWVALU for the first two passes; VALU may
+ // co-execute with MFMA starting at pass 3.
+ let ReleaseAtCycles = [2, 8] in
+ def : HWWriteRes<Write8PassMAI, [HWVALU, HWXDL], 8>;
let ReleaseAtCycles = [16] in
def : HWWriteRes<Write16PassMAI, [HWXDL], 16>;
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 727728145dfe0..601a09f1dfdea 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -370,76 +370,151 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX908-LABEL: v32_asm_def_use:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v33, v1
-; GFX908-NEXT: v_mov_b32_e32 v34, v0
+; GFX908-NEXT: v_mov_b32_e32 v32, v1
+; GFX908-NEXT: v_mov_b32_e32 v33, v0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def v[0:31] a[0:15]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a15
-; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def v32
-; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a14
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a14
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a13
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a13
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a12
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a12
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a11
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a11
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a10
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a9
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a8
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a7
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a6
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a5
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a4
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a3
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v35
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX908-NEXT: s_nop 9
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX908-NEXT: ;;#ASMSTART
+; GFX908-NEXT: ; def v32
+; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
-; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_write_b32 a32, v35
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: s_nop 7
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v35
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; use a3 v[0:31]
; GFX908-NEXT: ;;#ASMEND
@@ -451,9 +526,8 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-LABEL: v32_asm_def_use:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_read_b32 v35, a32 ; Reload Reuse
-; GFX90A-NEXT: v_mov_b32_e32 v34, v0
-; GFX90A-NEXT: v_mov_b32_e32 v33, v1
+; GFX90A-NEXT: v_mov_b32_e32 v33, v0
+; GFX90A-NEXT: v_mov_b32_e32 v32, v1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:31] a[0:15]
; GFX90A-NEXT: ;;#ASMEND
@@ -473,20 +547,53 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-NEXT: v_accvgpr_mov_b32 a18, a2
; GFX90A-NEXT: v_accvgpr_mov_b32 a17, a1
; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a0
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX90A-NEXT: s_nop 10
+; GFX90A-NEXT: buffer_store_dword a0, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v32
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
-; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
+; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a1
+; GFX90A-NEXT: buffer_load_dword a0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_write_b32 a32, v35 ; Reload Reuse
-; GFX90A-NEXT: s_nop 9
; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a3 v[0:31]
@@ -1140,9 +1247,9 @@ define void @no_free_vgprs_at_sgpr_to_agpr_copy(float %v0, float %v1) #0 {
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32 immarg, i32 immarg, i32 immarg) #1
declare i32 @llvm.amdgcn.workitem.id.x() #2
-attributes #0 = { "amdgpu-waves-per-eu"="6,6" nounwind }
+attributes #0 = { "amdgpu-waves-per-eu"="6,6" }
attributes #1 = { convergent nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
-attributes #3 = { "amdgpu-waves-per-eu"="7,7" "amdgpu-agpr-alloc"="0" nounwind }
-attributes #4 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-flat-work-group-size"="1024,1024" nounwind }
-attributes #5 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-agpr-alloc"="0" nounwind }
+attributes #3 = { "amdgpu-waves-per-eu"="7,7" "amdgpu-agpr-alloc"="0" }
+attributes #4 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-flat-work-group-size"="1024,1024" }
+attributes #5 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-agpr-alloc"="0" }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
index da92da4de0834..0ee6b32a83417 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
@@ -6,195 +6,201 @@
define amdgpu_kernel void @largeInterleave() #0 { ret void }
; GCN-LABEL: largeInterleave:
; GCN: ; %bb.0:
- ; GCN-NEXT: ; implicit-def: $vgpr16
- ; GCN-NEXT: ; implicit-def: $vgpr25
+ ; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
- ; GCN-NEXT: v_readfirstlane_b32 s17, v16
+ ; GCN-NEXT: v_readfirstlane_b32 s18, v0
+ ; GCN-NEXT: ; implicit-def: $vgpr9
; GCN-NEXT: ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
- ; GCN-NEXT: ; implicit-def: $vgpr17
+ ; GCN-NEXT: ; implicit-def: $vgpr1
+ ; GCN-NEXT: ; implicit-def: $vgpr4
+ ; GCN-NEXT: ; implicit-def: $sgpr16
; GCN-NEXT: ; implicit-def: $sgpr15
+ ; GCN-NEXT: ; implicit-def: $vgpr5
; GCN-NEXT: ; implicit-def: $sgpr8_sgpr9_sgpr10_sgpr11
- ; GCN-NEXT: s_lshl_b32 s18, s17, 7
- ; GCN-NEXT: ; implicit-def: $vgpr18
- ; GCN-NEXT: v_add_lshl_u32 v230, v18, s18, 1
- ; GCN-NEXT: v_lshl_add_u32 v25, s17, 4, v25
- ; GCN-NEXT: v_mul_lo_u32 v25, v25, s6
- ; GCN-NEXT: v_add_lshl_u32 v226, v25, v17, 1
- ; GCN-NEXT: v_add_u32_e32 v17, s15, v226
- ; GCN-NEXT: buffer_load_dwordx4 v[64:67], v226, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: ; implicit-def: $vgpr2
+ ; GCN-NEXT: ; implicit-def: $vgpr238
+ ; GCN-NEXT: ; implicit-def: $vgpr239
+ ; GCN-NEXT: ; implicit-def: $vgpr128_vgpr129_vgpr130_vgpr131
+ ; GCN-NEXT: ; implicit-def: $vgpr132_vgpr133_vgpr134_vgpr135
+ ; GCN-NEXT: ; implicit-def: $vgpr154_vgpr155_vgpr156_vgpr157
+ ; GCN-NEXT: ; implicit-def: $vgpr158_vgpr159_vgpr160_vgpr161
+ ; GCN-NEXT: ; implicit-def: $vgpr3
+ ; GCN-NEXT: ; implicit-def: $vgpr10
+ ; GCN-NEXT: ; implicit-def: $vgpr11
+ ; GCN-NEXT: ; implicit-def: $vgpr12
+ ; GCN-NEXT: ; implicit-def: $vgpr13
+ ; GCN-NEXT: v_add_u32_e32 v232, v3, v10
+ ; GCN-NEXT: v_add_u32_e32 v233, v3, v11
+ ; GCN-NEXT: v_lshl_add_u32 v0, s18, 4, v9
+ ; GCN-NEXT: v_mul_lo_u32 v0, v0, s6
+ ; GCN-NEXT: v_add_lshl_u32 v230, v0, v1, 1
+ ; GCN-NEXT: v_add_u32_e32 v0, s18, v4
+ ; GCN-NEXT: v_and_b32_e32 v0, 0x1fffffff, v0
+ ; GCN-NEXT: v_mul_lo_u32 v0, v0, s16
+ ; GCN-NEXT: v_add_lshl_u32 v140, v5, v0, 1
+ ; GCN-NEXT: v_add_u32_e32 v0, s15, v230
+ ; GCN-NEXT: buffer_load_dwordx4 v[64:67], v230, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx4 v[68:71], v17, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[68:71], v0, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_add_u32_e32 v72, 64, v17
- ; GCN-NEXT: ; implicit-def: $vgpr213
- ; GCN-NEXT: ; implicit-def: $vgpr152_vgpr153_vgpr154_vgpr155
- ; GCN-NEXT: ; implicit-def: $vgpr246
- ; GCN-NEXT: v_add_u32_e32 v188, 0x80, v17
- ; GCN-NEXT: ; implicit-def: $vgpr156_vgpr157_vgpr158_vgpr159
- ; GCN-NEXT: ; implicit-def: $vgpr144_vgpr145_vgpr146_vgpr147
- ; GCN-NEXT: ; implicit-def: $vgpr19
- ; GCN-NEXT: ; implicit-def: $vgpr26
- ; GCN-NEXT: ; implicit-def: $vgpr27
- ; GCN-NEXT: v_add_u32_e32 v227, 0xc0, v17
- ; GCN-NEXT: v_add_u32_e32 v231, v19, v26
- ; GCN-NEXT: v_add_u32_e32 v232, v19, v27
+ ; GCN-NEXT: s_lshl_b32 s6, s18, 7
+ ; GCN-NEXT: v_add_lshl_u32 v240, v2, s6, 1
+ ; GCN-NEXT: v_add_u32_e32 v72, 64, v0
+ ; GCN-NEXT: v_add_u32_e32 v92, 0x80, v0
+ ; GCN-NEXT: v_add_u32_e32 v231, 0xc0, v0
+ ; GCN-NEXT: v_add_u32_e32 v234, v3, v12
+ ; GCN-NEXT: v_add_u32_e32 v235, v3, v13
; GCN-NEXT: ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3
- ; GCN-NEXT: ; implicit-def: $vgpr28
- ; GCN-NEXT: ; implicit-def: $vgpr29
- ; GCN-NEXT: v_add_u32_e32 v233, v19, v28
- ; GCN-NEXT: v_add_u32_e32 v234, v19, v29
- ; GCN-NEXT: ; implicit-def: $vgpr140_vgpr141_vgpr142_vgpr143
+ ; GCN-NEXT: ; implicit-def: $vgpr162_vgpr163_vgpr164_vgpr165
+ ; GCN-NEXT: ; implicit-def: $vgpr166_vgpr167_vgpr168_vgpr169
+ ; GCN-NEXT: ; implicit-def: $vgpr170_vgpr171_vgpr172_vgpr173
+ ; GCN-NEXT: ; implicit-def: $vgpr174_vgpr175_vgpr176_vgpr177
+ ; GCN-NEXT: ; implicit-def: $sgpr14
; GCN-NEXT: ; implicit-def: $sgpr5
; GCN-NEXT: ; implicit-def: $sgpr7
- ; GCN-NEXT: ; implicit-def: $vgpr148_vgpr149_vgpr150_vgpr151
- ; GCN-NEXT: ; implicit-def: $vgpr136_vgpr137_vgpr138_vgpr139
- ; GCN-NEXT: ; implicit-def: $vgpr132_vgpr133_vgpr134_vgpr135
+ ; GCN-NEXT: ; implicit-def: $vgpr6
+ ; GCN-NEXT: v_lshl_add_u32 v141, v6, 1, v140
+ ; GCN-NEXT: ; implicit-def: $vgpr7
+ ; GCN-NEXT: v_lshl_add_u32 v142, v7, 1, v141
+ ; GCN-NEXT: ; implicit-def: $vgpr8
+ ; GCN-NEXT: v_lshl_add_u32 v143, v8, 1, v142
; GCN-NEXT: ; implicit-def: $vgpr20
- ; GCN-NEXT: v_add_u32_e32 v18, s17, v20
- ; GCN-NEXT: v_and_b32_e32 v18, 0x1fffffff, v18
- ; GCN-NEXT: ; implicit-def: $sgpr16
- ; GCN-NEXT: v_mul_lo_u32 v18, v18, s16
- ; GCN-NEXT: ; implicit-def: $vgpr21
- ; GCN-NEXT: v_add_lshl_u32 v199, v21, v18, 1
- ; GCN-NEXT: ; implicit-def: $vgpr22
- ; GCN-NEXT: v_lshl_add_u32 v200, v22, 1, v199
- ; GCN-NEXT: ; implicit-def: $vgpr23
- ; GCN-NEXT: v_lshl_add_u32 v201, v23, 1, v200
- ; GCN-NEXT: ; implicit-def: $vgpr24
- ; GCN-NEXT: v_lshl_add_u32 v202, v24, 1, v201
+ ; GCN-NEXT: v_add_u32_e32 v241, v3, v20
+ ; GCN-NEXT: ; implicit-def: $vgpr14
+ ; GCN-NEXT: ; implicit-def: $vgpr15
; GCN-NEXT: ; implicit-def: $vgpr16
- ; GCN-NEXT: ; implicit-def: $vgpr18
- ; GCN-NEXT: ; implicit-def: $vgpr20
- ; GCN-NEXT: ; implicit-def: $vgpr24
- ; GCN-NEXT: v_add_u32_e32 v247, v19, v24
- ; GCN-NEXT: v_add_u32_e32 v248, v19, v16
- ; GCN-NEXT: v_add_u32_e32 v249, v19, v18
- ; GCN-NEXT: v_add_u32_e32 v250, v19, v20
- ; GCN-NEXT: ; implicit-def: $vgpr128_vgpr129_vgpr130_vgpr131
- ; GCN-NEXT: ; implicit-def: $sgpr14
- ; GCN-NEXT: ; implicit-def: $vgpr196
+ ; GCN-NEXT: v_add_u32_e32 v242, v3, v14
+ ; GCN-NEXT: v_add_u32_e32 v243, v3, v15
+ ; GCN-NEXT: v_add_u32_e32 v244, v3, v16
+ ; GCN-NEXT: ; implicit-def: $vgpr136
; GCN-NEXT: ; implicit-def: $sgpr12_sgpr13
- ; GCN-NEXT: ; implicit-def: $vgpr211
- ; GCN-NEXT: v_max_f32_e32 v212, v211, v211
- ; GCN-NEXT: ; implicit-def: $vgpr198
- ; GCN-NEXT: ; implicit-def: $vgpr32
- ; GCN-NEXT: ; implicit-def: $vgpr33
- ; GCN-NEXT: ; implicit-def: $vgpr34
- ; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; GCN-NEXT: v_add_u32_e32 v210, v19, v34
- ; GCN-NEXT: v_add_u32_e32 v206, v19, v33
- ; GCN-NEXT: v_add_u32_e32 v205, v19, v32
- ; GCN-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+ ; GCN-NEXT: ; implicit-def: $vgpr153
+ ; GCN-NEXT: v_max_f32_e32 v148, v153, v153
+ ; GCN-NEXT: ; implicit-def: $vgpr139
+ ; GCN-NEXT: ; implicit-def: $vgpr17
+ ; GCN-NEXT: ; implicit-def: $vgpr18
+ ; GCN-NEXT: ; implicit-def: $vgpr19
; GCN-NEXT: ; implicit-def: $vgpr21
; GCN-NEXT: ; implicit-def: $vgpr22
; GCN-NEXT: ; implicit-def: $vgpr23
- ; GCN-NEXT: ; implicit-def: $vgpr30
- ; GCN-NEXT: ; implicit-def: $vgpr31
- ; GCN-NEXT: v_add_u32_e32 v207, v19, v21
- ; GCN-NEXT: v_add_u32_e32 v208, v19, v22
- ; GCN-NEXT: v_add_u32_e32 v209, v19, v23
- ; GCN-NEXT: v_add_u32_e32 v203, v19, v30
- ; GCN-NEXT: v_add_u32_e32 v204, v19, v31
- ; GCN-NEXT: ; kill: killed $vgpr17
- ; GCN-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; GCN-NEXT: ; implicit-def: $vgpr24
+ ; GCN-NEXT: ; implicit-def: $vgpr25
+ ; GCN-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+ ; GCN-NEXT: ; implicit-def: $sgpr17
+ ; GCN-NEXT: v_add_u32_e32 v20, s17, v20
+ ; GCN-NEXT: v_add_u32_e32 v16, s17, v16
+ ; GCN-NEXT: v_add_u32_e32 v152, v3, v25
+ ; GCN-NEXT: v_add_u32_e32 v25, s17, v25
+ ; GCN-NEXT: v_add_u32_e32 v149, v3, v17
+ ; GCN-NEXT: v_add_u32_e32 v17, s17, v17
+ ; GCN-NEXT: v_add_u32_e32 v150, v3, v18
+ ; GCN-NEXT: v_add_u32_e32 v18, s17, v18
+ ; GCN-NEXT: v_add_u32_e32 v151, v3, v19
+ ; GCN-NEXT: v_add_u32_e32 v19, s17, v19
+ ; GCN-NEXT: v_add_u32_e32 v147, v3, v24
+ ; GCN-NEXT: v_add_u32_e32 v24, s17, v24
+ ; GCN-NEXT: v_add_u32_e32 v145, v3, v21
+ ; GCN-NEXT: v_add_u32_e32 v21, s17, v21
+ ; GCN-NEXT: v_add_u32_e32 v144, v3, v22
+ ; GCN-NEXT: v_add_u32_e32 v22, s17, v22
+ ; GCN-NEXT: v_add_u32_e32 v146, v3, v23
+ ; GCN-NEXT: v_add_u32_e32 v23, s17, v23
; GCN-NEXT: ; implicit-def: $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
- ; GCN-NEXT: ; implicit-def: $vgpr197
+ ; GCN-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+ ; GCN-NEXT: ; implicit-def: $vgpr138
+ ; GCN-NEXT: ; implicit-def: $vgpr137
; GCN-NEXT: ; iglp_opt mask(0x00000002)
+ ; GCN-NEXT: ; kill: killed $vgpr145
+ ; GCN-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GCN-NEXT: ; kill: killed $vgpr146
+ ; GCN-NEXT: ; kill: killed $vgpr144
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[64:67]
+ ; GCN-NEXT: ds_write_b128 v240, v[64:67]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[68:71] offset:1024
+ ; GCN-NEXT: ds_write_b128 v240, v[68:71] offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx4 v[160:163], v226, s[8:11], 0 offen offset:64 sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[80:83], v230, s[8:11], 0 offen offset:64 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx4 v[164:167], v72, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[84:87], v72, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: ds_read_b128 v[64:67], v213
+ ; GCN-NEXT: ds_read_b128 v[64:67], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[64:65], v[152:153], 0
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[66:67], v[154:155], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[64:67], v213 offset:512
+ ; GCN-NEXT: ds_read_b128 v[68:71], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[64:65], v[152:153], 0
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[66:67], v[154:155], v[96:111]
- ; GCN-NEXT: ds_read_b128 v[64:67], v213 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[178:181], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[168:171], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[88:91], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[172:175], v246
+ ; GCN-NEXT: ds_read_b128 v[72:75], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[176:179], v246 offset:512
+ ; GCN-NEXT: ds_read_b128 v[76:79], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[180:183], v246 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[182:185], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[184:187], v246 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[64:65], v[152:153], 0
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[64:65], v[128:129], 0
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[160:163]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[66:67], v[154:155], v[80:95]
+ ; GCN-NEXT: ds_write_b128 v240, v[80:83]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[164:167] offset:1024
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[168:169], v[152:153], 0
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[170:171], v[154:155], v[64:79]
+ ; GCN-NEXT: ds_write_b128 v240, v[84:87] offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx4 v[152:155], v226, s[8:11], 0 offen offset:128 sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[190:193], v230, s[8:11], 0 offen offset:128 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx4 v[160:163], v188, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[194:197], v92, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[68:69], v[128:129], 0
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: ds_read_b128 v[188:191], v213
+ ; GCN-NEXT: ds_read_b128 v[198:201], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[192:195], v213 offset:512
+ ; GCN-NEXT: ds_read_b128 v[202:205], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[164:167], v213 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[206:209], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[214:217], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[210:213], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[172:173], v[156:157], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[218:221], v246
+ ; GCN-NEXT: ds_read_b128 v[214:217], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[222:225], v246 offset:512
+ ; GCN-NEXT: ds_read_b128 v[218:221], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[168:171], v246 offset:1024
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[66:67], v[130:131], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[222:225], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[174:175], v[158:159], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[188:189], v[144:145], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[190:191], v[146:147], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[188:191], v246 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[226:229], v239 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -202,81 +208,100 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[152:155]
+ ; GCN-NEXT: ds_write_b128 v240, v[190:193]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[70:71], v[130:131], v[96:111]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[160:163] offset:1024
+ ; GCN-NEXT: ds_write_b128 v240, v[194:197] offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx4 v[152:155], v226, s[8:11], 0 offen offset:192 sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[190:193], v230, s[8:11], 0 offen offset:192 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[184:185], v[156:157], v[64:79]
- ; GCN-NEXT: buffer_load_dwordx4 v[226:229], v227, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[194:197], v231, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[160:161], v231, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[72:73], v[132:133], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[76:77], v[132:133], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[74:75], v[134:135], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[78:79], v[134:135], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[88:89], v[128:129], 0
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[90:91], v[130:131], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[178:179], v[128:129], 0
+ ; GCN-NEXT: buffer_load_dwordx2 v[128:129], v232, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[162:163], v232, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[178:179], v233, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[172:173], v233, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[230:231], v234, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[174:175], v234, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v234, v178, v128, s5
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[198:199], v[154:155], v[112:127]
+ ; GCN-NEXT: buffer_load_dwordx2 v[232:233], v235, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[158:159], v[64:79]
- ; GCN-NEXT: v_perm_b32 v238, v162, v160, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[218:219], v[140:141], v[112:127]
- ; GCN-NEXT: v_perm_b32 v240, v162, v160, s7
- ; GCN-NEXT: v_perm_b32 v242, v163, v161, s5
- ; GCN-NEXT: v_perm_b32 v244, v163, v161, s7
- ; GCN-NEXT: ds_read_b128 v[160:163], v213
+ ; GCN-NEXT: v_perm_b32 v236, v178, v128, s7
+ ; GCN-NEXT: v_perm_b32 v128, v179, v129, s5
+ ; GCN-NEXT: v_perm_b32 v178, v179, v129, s7
+ ; GCN-NEXT: v_perm_b32 v235, v232, v230, s5
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[132:133], v[64:79]
+ ; GCN-NEXT: v_perm_b32 v237, v232, v230, s7
+ ; GCN-NEXT: v_perm_b32 v129, v233, v231, s5
+ ; GCN-NEXT: v_perm_b32 v179, v233, v231, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[180:181], v[130:131], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[200:201], v[156:157], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[198:201], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_perm_b32 v239, v174, v172, s5
- ; GCN-NEXT: v_perm_b32 v241, v174, v172, s7
- ; GCN-NEXT: v_perm_b32 v243, v175, v173, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[214:215], v[144:145], v[64:79]
- ; GCN-NEXT: v_perm_b32 v245, v175, v173, s7
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[176:177], v[156:157], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[220:221], v[142:143], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[218:221], v213 offset:512
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[134:135], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[182:183], v[132:133], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[214:215], v[158:159], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[210:211], v[154:155], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[202:203], v[154:155], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[184:185], v[134:135], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[216:217], v[160:161], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[214:217], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[172:175], v213 offset:1024
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[212:213], v[156:157], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[204:205], v[156:157], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[206:207], v[154:155], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[198:199], v[162:163], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[226:227], v[158:159], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[218:219], v[158:159], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[208:209], v[156:157], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[200:201], v[164:165], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[198:201], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[216:217], v[146:147], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[178:179], v[158:159], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[160:161], v[148:149], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[140:141], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[192:193], v[144:145], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[162:163], v[150:151], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[160:163], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[186:189], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[184:187], v246
+ ; GCN-NEXT: ds_read_b128 v[210:213], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[214:217], v246 offset:512
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[228:229], v[160:161], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[220:221], v[160:161], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[222:223], v[158:159], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[162:163], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[214:215], v[162:163], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[224:225], v[160:161], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[210:211], v[166:167], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[164:165], v[64:79]
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[176:179], v246 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[226:229], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[190:191], v[142:143], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[194:195], v[146:147], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[160:161], v[148:149], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[180:181], v[156:157], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[184:185], v[136:137], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[222:223], v[140:141], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[162:163], v[150:151], v[64:79]
- ; GCN-NEXT: ds_read_b128 v[160:163], v246 offset:1536
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[216:217], v[164:165], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[198:199], v[162:163], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[212:213], v[168:169], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[210:213], v239 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -284,278 +309,259 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[152:155]
+ ; GCN-NEXT: ds_write_b128 v240, v[190:193]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[226:229] offset:1024
+ ; GCN-NEXT: ds_write_b128 v240, v[194:197] offset:1024
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[182:183], v[158:159], v[80:95]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[156:159], v213
+ ; GCN-NEXT: ds_read_b128 v[190:193], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[226:229], v213 offset:512
+ ; GCN-NEXT: ds_read_b128 v[194:197], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[180:183], v213 offset:1024
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[186:187], v[166:167], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[202:205], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[152:155], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[218:221], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[230:233], v246
+ ; GCN-NEXT: ds_read_b128 v[214:217], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[234:237], v246 offset:512
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[200:201], v[164:165], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[188:189], v[168:169], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[186:187], v[138:139], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[184:187], v246 offset:1024
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[224:225], v[142:143], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[156:157], v[132:133], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[218:219], v[148:149], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[158:159], v[134:135], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[156:159], v246 offset:1536
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[190:191], v[170:171], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[226:227], v[166:167], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[194:195], v[170:171], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[192:193], v[172:173], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[228:229], v[168:169], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[210:211], v[166:167], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[196:197], v[172:173], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[194:197], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[214:215], v[174:175], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[202:203], v[170:171], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[212:213], v[168:169], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[186:187], v[174:175], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[216:217], v[176:177], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[204:205], v[172:173], v[80:95]
+ ; GCN-NEXT: s_nop 9
+ ; GCN-NEXT: v_mul_f32_e32 v154, s4, v112
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v113
+ ; GCN-NEXT: v_max3_f32 v154, v154, s14, v155
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v114
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v115
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[218:219], v[170:171], v[64:79]
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v116
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v117
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v118
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v119
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[188:189], v[176:177], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:1536
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v120
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v121
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v122
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v123
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[194:195], v[174:175], v[80:95]
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v124
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v125
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v126
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v127
+ ; GCN-NEXT: v_mul_f32_e32 v190, s4, v96
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[220:221], v[172:173], v[64:79]
+ ; GCN-NEXT: v_mul_f32_e32 v191, s4, v97
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[238:239]
+ ; GCN-NEXT: ds_write_b64 v140, v[234:235]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[240:241]
+ ; GCN-NEXT: ds_write_b64 v141, v[236:237]
+ ; GCN-NEXT: v_mul_f32_e32 v236, s4, v98
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[196:197], v[176:177], v[80:95]
+ ; GCN-NEXT: v_mul_f32_e32 v237, s4, v99
+ ; GCN-NEXT: v_max3_f32 v154, v154, v190, v191
+ ; GCN-NEXT: v_mul_f32_e32 v238, s4, v100
+ ; GCN-NEXT: v_mul_f32_e32 v239, s4, v101
+ ; GCN-NEXT: v_max3_f32 v154, v154, v236, v237
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[242:243]
+ ; GCN-NEXT: ds_write_b64 v142, v[128:129]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[174:175], v[64:79]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[244:245]
+ ; GCN-NEXT: ds_write_b64 v143, v[178:179]
+ ; GCN-NEXT: v_mul_f32_e32 v192, s4, v102
+ ; GCN-NEXT: v_mul_f32_e32 v193, s4, v103
+ ; GCN-NEXT: v_max3_f32 v154, v154, v238, v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx2 v[192:193], v247, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[178:179], v241, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[220:221], v[150:151], v[96:111]
- ; GCN-NEXT: buffer_load_dwordx2 v[194:195], v248, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v240, s4, v104
+ ; GCN-NEXT: v_mul_f32_e32 v241, s4, v105
+ ; GCN-NEXT: v_max3_f32 v154, v154, v192, v193
+ ; GCN-NEXT: buffer_load_dwordx2 v[230:231], v242, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[218:219], v249, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[232:233], v243, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[220:221], v250, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[234:235], v244, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v242, s4, v106
+ ; GCN-NEXT: v_max3_f32 v154, v154, v240, v241
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v107
+ ; GCN-NEXT: v_max3_f32 v154, v154, v242, v155
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v108
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v109
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v110
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v111
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[176:177], v[64:79]
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v80
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v81
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v82
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v83
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v84
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v85
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v86
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v87
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v88
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v89
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v90
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v91
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v92
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v93
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v94
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v95
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v64
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v65
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v66
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v67
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v68
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v69
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v70
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v71
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v72
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v73
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v74
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v75
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v76
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v77
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v78
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v79
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: ds_bpermute_b32 v155, v136, v154
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_perm_b32 v188, v194, v192, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[164:165], v[144:145], v[80:95]
- ; GCN-NEXT: v_perm_b32 v189, v220, v218, s5
- ; GCN-NEXT: v_perm_b32 v191, v220, v218, s7
- ; GCN-NEXT: v_perm_b32 v190, v194, v192, s7
- ; GCN-NEXT: v_perm_b32 v192, v195, v193, s5
- ; GCN-NEXT: v_perm_b32 v194, v195, v193, s7
- ; GCN-NEXT: v_perm_b32 v193, v221, v219, s5
- ; GCN-NEXT: v_perm_b32 v195, v221, v219, s7
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[166:167], v[146:147], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[168:169], v[140:141], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[170:171], v[142:143], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[172:173], v[148:149], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[214:215], v[136:137], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[174:175], v[150:151], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[216:217], v[138:139], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[176:177], v[136:137], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[226:227], v[132:133], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[178:179], v[138:139], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[160:161], v[136:137], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[230:231], v[128:129], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[228:229], v[134:135], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[180:181], v[132:133], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[162:163], v[138:139], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[232:233], v[130:131], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[234:235], v[128:129], v[96:111]
- ; GCN-NEXT: s_nop 9
- ; GCN-NEXT: v_mul_f32_e32 v213, s4, v112
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v113
- ; GCN-NEXT: v_max3_f32 v213, v213, s14, v218
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v114
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v115
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v116
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[182:183], v[134:135], v[80:95]
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v117
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v118
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v119
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v120
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v121
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[152:153], v[132:133], v[64:79]
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v122
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v123
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v124
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v125
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[236:237], v[130:131], v[96:111]
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v126
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v127
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[184:185], v[128:129], v[80:95]
- ; GCN-NEXT: s_nop 6
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v96
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v97
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v98
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v99
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v100
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[154:155], v[134:135], v[64:79]
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v101
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v102
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v103
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v104
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v105
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[186:187], v[130:131], v[80:95]
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v106
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v107
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v108
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v109
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[156:157], v[128:129], v[64:79]
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v110
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v111
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v140, s4, v80
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v81
- ; GCN-NEXT: v_max3_f32 v140, v213, v140, v141
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v82
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[158:159], v[130:131], v[64:79]
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v83
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v84
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v85
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v86
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v87
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v88
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v89
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v90
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v91
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v92
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v93
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v94
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v95
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v128, s4, v64
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v65
- ; GCN-NEXT: v_max3_f32 v128, v140, v128, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v66
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v67
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v68
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v69
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v70
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v71
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v72
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v73
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v74
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v75
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v76
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v77
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v78
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v79
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: ds_bpermute_b32 v129, v196, v128
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[130:133], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_max_f32_e32 v129, v129, v129
- ; GCN-NEXT: v_max_f32_e32 v128, v128, v129
- ; GCN-NEXT: ds_bpermute_b32 v129, v196, v128
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: v_cndmask_b32_e64 v128, v129, v128, s[12:13]
- ; GCN-NEXT: v_max_f32_e32 v128, v128, v128
- ; GCN-NEXT: v_max_f32_e32 v128, v212, v128
- ; GCN-NEXT: v_fma_f32 v113, s4, v113, -v128
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v114, -v128
- ; GCN-NEXT: v_mul_f32_e32 v139, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v115, -v128
- ; GCN-NEXT: v_mul_f32_e32 v140, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v116, -v128
- ; GCN-NEXT: v_mul_f32_e32 v141, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v117, -v128
- ; GCN-NEXT: v_fma_f32 v112, s4, v112, -v128
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v118, -v128
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_max_f32_e32 v155, v155, v155
+ ; GCN-NEXT: v_max_f32_e32 v154, v154, v155
+ ; GCN-NEXT: ds_bpermute_b32 v155, v136, v154
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_cndmask_b32_e64 v154, v155, v154, s[12:13]
+ ; GCN-NEXT: v_max_f32_e32 v154, v154, v154
+ ; GCN-NEXT: v_max_f32_e32 v148, v148, v154
+ ; GCN-NEXT: v_fma_f32 v112, s4, v112, -v148
; GCN-NEXT: v_mul_f32_e32 v112, 0x3fb8aa3b, v112
- ; GCN-NEXT: v_mul_f32_e32 v143, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v119, -v128
- ; GCN-NEXT: v_mul_f32_e32 v144, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_exp_f32_e32 v113, v112
- ; GCN-NEXT: v_exp_f32_e32 v114, v138
- ; GCN-NEXT: v_exp_f32_e32 v115, v139
- ; GCN-NEXT: v_exp_f32_e32 v116, v140
- ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v113
- ; GCN-NEXT: v_cvt_f16_f32_e32 v119, v114
- ; GCN-NEXT: v_fma_f32 v118, s4, v120, -v128
- ; GCN-NEXT: v_fma_f32 v120, s4, v121, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v121, v116
- ; GCN-NEXT: v_pack_b32_f16 v146, v112, v119
- ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v115
- ; GCN-NEXT: v_sub_f32_e32 v129, v211, v128
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v129
- ; GCN-NEXT: v_mul_f32_e32 v149, 0x3fb8aa3b, v120
- ; GCN-NEXT: v_fma_f32 v120, s4, v122, -v128
- ; GCN-NEXT: v_exp_f32_e32 v117, v141
- ; GCN-NEXT: v_mul_f32_e32 v148, 0x3fb8aa3b, v118
- ; GCN-NEXT: v_exp_f32_e32 v118, v142
- ; GCN-NEXT: v_exp_f32_e32 v119, v143
- ; GCN-NEXT: v_mul_f32_e32 v150, 0x3fb8aa3b, v120
- ; GCN-NEXT: v_exp_f32_e32 v120, v144
- ; GCN-NEXT: v_pack_b32_f16 v147, v112, v121
- ; GCN-NEXT: v_exp_f32_e32 v112, v129
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_fma_f32 v122, s4, v123, -v128
- ; GCN-NEXT: v_mul_f32_e32 v151, 0x3fb8aa3b, v122
- ; GCN-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[10:11], v[10:11], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[12:13], v[12:13], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[14:15], v[14:15], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_fma_f32 v113, s4, v113, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v113, 0x3fb8aa3b, v113
+ ; GCN-NEXT: v_fma_f32 v114, s4, v114, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v112, v112
+ ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v115, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v113, v113
+ ; GCN-NEXT: v_mul_f32_e32 v159, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v116, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v158, v158
+ ; GCN-NEXT: v_mul_f32_e32 v160, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v117, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v159, v159
+ ; GCN-NEXT: v_mul_f32_e32 v161, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_add_f32_e32 v162, 0, v112
+ ; GCN-NEXT: v_exp_f32_e32 v164, v160
+ ; GCN-NEXT: v_add_f32_e32 v162, v113, v162
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v112
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v113, v113
+ ; GCN-NEXT: v_exp_f32_e32 v166, v161
+ ; GCN-NEXT: v_add_f32_e32 v162, v158, v162
+ ; GCN-NEXT: v_fma_f32 v114, s4, v118, -v148
+ ; GCN-NEXT: v_add_f32_e32 v162, v159, v162
+ ; GCN-NEXT: v_fma_f32 v120, s4, v120, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v118, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v119, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v165, 0x3fb8aa3b, v120
+ ; GCN-NEXT: v_fma_f32 v120, s4, v121, -v148
+ ; GCN-NEXT: v_add_f32_e32 v121, v164, v162
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v114
+ ; GCN-NEXT: ds_read_b128 v[114:117], v139
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v162, v112, v113
+ ; GCN-NEXT: v_mul_f32_e32 v113, 0x3fb8aa3b, v120
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v158
+ ; GCN-NEXT: v_add_f32_e32 v120, v166, v121
+ ; GCN-NEXT: v_fma_f32 v121, s4, v122, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v122, v118
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v118, v159
+ ; GCN-NEXT: ds_read_b128 v[154:157], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v167, 0x3fb8aa3b, v121
+ ; GCN-NEXT: v_fma_f32 v121, s4, v123, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v123, v119
+ ; GCN-NEXT: v_sub_f32_e32 v153, v153, v148
+ ; GCN-NEXT: v_mul_f32_e32 v153, 0x3fb8aa3b, v153
+ ; GCN-NEXT: v_add_f32_e32 v120, v122, v120
+ ; GCN-NEXT: v_pack_b32_f16 v163, v112, v118
+ ; GCN-NEXT: v_exp_f32_e32 v112, v153
+ ; GCN-NEXT: v_mul_f32_e32 v168, 0x3fb8aa3b, v121
+ ; GCN-NEXT: v_add_f32_e32 v153, v123, v120
+ ; GCN-NEXT: ds_read_b128 v[118:121], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[158:161], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: v_pk_mul_f32 v[32:33], v[32:33], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[34:35], v[34:35], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[36:37], v[36:37], v[112:113] op_sel_hi:[1,0]
@@ -564,234 +570,268 @@
; GCN-NEXT: v_pk_mul_f32 v[42:43], v[42:43], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[44:45], v[44:45], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[46:47], v[46:47], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v123, v117
- ; GCN-NEXT: v_fma_f32 v122, s4, v124, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v124, v118
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[146:147], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v121, v148
- ; GCN-NEXT: v_pk_mul_f32 v[16:17], v[16:17], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[18:19], v[18:19], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[20:21], v[20:21], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[22:23], v[22:23], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[24:25], v[24:25], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[26:27], v[26:27], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[134:135], v[146:147], v[32:47]
- ; GCN-NEXT: v_mul_f32_e64 v28, v28, v112
- ; GCN-NEXT: v_mul_f32_e64 v29, v29, v112
- ; GCN-NEXT: v_mul_f32_e64 v30, v30, v112
- ; GCN-NEXT: v_mul_f32_e64 v31, v31, v112
- ; GCN-NEXT: v_mul_f32_e64 v48, v48, v112
- ; GCN-NEXT: v_mul_f32_e64 v49, v49, v112
+ ; GCN-NEXT: v_pk_mul_f32 v[48:49], v[48:49], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[50:51], v[50:51], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[114:115], v[162:163], v[32:47]
; GCN-NEXT: v_pk_mul_f32 v[52:53], v[52:53], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[54:55], v[54:55], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[56:57], v[56:57], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[58:59], v[58:59], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[60:61], v[60:61], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[62:63], v[62:63], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pack_b32_f16 v134, v123, v124
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v119
- ; GCN-NEXT: v_fma_f32 v124, s4, v126, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v120
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v122
- ; GCN-NEXT: v_exp_f32_e32 v122, v149
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[138:139], v[146:147], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v123, v150
- ; GCN-NEXT: v_pack_b32_f16 v135, v130, v126
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v124
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v121
- ; GCN-NEXT: v_fma_f32 v139, s4, v96, -v128
- ; GCN-NEXT: v_fma_f32 v125, s4, v125, -v128
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v125
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[142:143], v[146:147], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v124, v151
- ; GCN-NEXT: v_fma_f32 v127, s4, v127, -v128
- ; GCN-NEXT: v_mul_f32_e32 v127, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_fma_f32 v143, s4, v101, -v128
- ; GCN-NEXT: v_fma_f32 v64, s4, v64, -v128
- ; GCN-NEXT: v_fma_f32 v65, s4, v65, -v128
- ; GCN-NEXT: v_fma_f32 v68, s4, v68, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[134:135], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v96, v129
- ; GCN-NEXT: ds_read_b128 v[130:133], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[146:149], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_fma_f32 v69, s4, v69, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[136:137], v[134:135], v[32:47]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v122
- ; GCN-NEXT: v_fma_f32 v137, s4, v97, -v128
- ; GCN-NEXT: v_exp_f32_e32 v97, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_pack_b32_f16 v126, v126, v136
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v123
- ; GCN-NEXT: v_fma_f32 v137, s4, v98, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[140:141], v[134:135], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v98, v138
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[144:145], v[134:135], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v124
- ; GCN-NEXT: v_fma_f32 v135, s4, v99, -v128
- ; GCN-NEXT: v_exp_f32_e32 v99, v127
- ; GCN-NEXT: v_mul_f32_e32 v150, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_pack_b32_f16 v127, v136, v134
- ; GCN-NEXT: ds_read_b128 v[134:137], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[138:141], v197 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[126:127], v[0:15]
- ; GCN-NEXT: v_fma_f32 v131, s4, v100, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v96
- ; GCN-NEXT: v_exp_f32_e32 v100, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_cvt_f16_f32_e32 v131, v97
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v114, v164
+ ; GCN-NEXT: v_exp_f32_e32 v164, v165
+ ; GCN-NEXT: v_pk_mul_f32 v[16:17], v[16:17], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[18:19], v[18:19], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[20:21], v[20:21], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[22:23], v[22:23], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[24:25], v[24:25], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[26:27], v[26:27], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[28:29], v[28:29], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[30:31], v[30:31], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[10:11], v[10:11], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[12:13], v[12:13], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[14:15], v[14:15], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_fma_f32 v115, s4, v124, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[154:155], v[162:163], v[48:63]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v124, v166
+ ; GCN-NEXT: v_exp_f32_e32 v113, v113
+ ; GCN-NEXT: v_mul_f32_e32 v165, 0x3fb8aa3b, v115
+ ; GCN-NEXT: v_add_f32_e32 v115, v164, v153
+ ; GCN-NEXT: v_pack_b32_f16 v154, v114, v124
+ ; GCN-NEXT: v_add_f32_e32 v114, v113, v115
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[118:119], v[162:163], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v115, v122
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v122, v123
+ ; GCN-NEXT: v_exp_f32_e32 v119, v167
+ ; GCN-NEXT: v_fma_f32 v123, s4, v127, -v148
+ ; GCN-NEXT: v_fma_f32 v125, s4, v125, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v155, v115, v122
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[158:159], v[162:163], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v127, v168
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v159, v164
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v113, v113
+ ; GCN-NEXT: v_mul_f32_e32 v153, 0x3fb8aa3b, v125
+ ; GCN-NEXT: v_fma_f32 v118, s4, v126, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v118, 0x3fb8aa3b, v118
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[116:117], v[154:155], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v162, v165
+ ; GCN-NEXT: v_add_f32_e32 v114, v119, v114
+ ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v123
+ ; GCN-NEXT: v_add_f32_e32 v126, v127, v114
+ ; GCN-NEXT: v_fma_f32 v96, s4, v96, -v148
+ ; GCN-NEXT: v_fma_f32 v97, s4, v97, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[156:157], v[154:155], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v153, v153
+ ; GCN-NEXT: v_fma_f32 v98, s4, v98, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v163, 0x3fb8aa3b, v96
+ ; GCN-NEXT: v_add_f32_e32 v96, v162, v126
+ ; GCN-NEXT: v_pack_b32_f16 v126, v159, v113
+ ; GCN-NEXT: v_mul_f32_e32 v113, 0x3fb8aa3b, v97
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[120:121], v[154:155], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v156, v118
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v119
+ ; GCN-NEXT: v_mul_f32_e32 v157, 0x3fb8aa3b, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v98, v127
+ ; GCN-NEXT: v_add_f32_e32 v96, v153, v96
+ ; GCN-NEXT: v_add_f32_e32 v96, v156, v96
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[160:161], v[154:155], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v154, v158
+ ; GCN-NEXT: v_fma_f32 v99, s4, v99, -v148
+ ; GCN-NEXT: ds_read_b128 v[114:117], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[122:125], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v127, v97, v98
+ ; GCN-NEXT: v_mul_f32_e32 v155, 0x3fb8aa3b, v99
+ ; GCN-NEXT: v_add_f32_e32 v158, v154, v96
+ ; GCN-NEXT: ds_read_b128 v[96:99], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[118:121], v138 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v128, v230, v178, s5
+ ; GCN-NEXT: v_perm_b32 v129, v234, v232, s5
+ ; GCN-NEXT: v_perm_b32 v130, v230, v178, s7
+ ; GCN-NEXT: v_perm_b32 v131, v234, v232, s7
+ ; GCN-NEXT: v_perm_b32 v132, v231, v179, s5
+ ; GCN-NEXT: v_perm_b32 v134, v231, v179, s7
+ ; GCN-NEXT: v_perm_b32 v133, v235, v233, s5
+ ; GCN-NEXT: v_perm_b32 v135, v235, v233, s7
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[188:189]
+ ; GCN-NEXT: ds_write_b64 v140, v[128:129]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[190:191]
+ ; GCN-NEXT: ds_write_b64 v141, v[130:131]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[192:193]
+ ; GCN-NEXT: ds_write_b64 v142, v[132:133]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[194:195]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[146:147], v[126:127], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v101, v125
- ; GCN-NEXT: v_pack_b32_f16 v146, v130, v131
+ ; GCN-NEXT: ds_write_b64 v143, v[134:135]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[114:115], v[126:127], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v132, v163
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx2 v[130:131], v210, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[128:129], v152, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v143
- ; GCN-NEXT: v_cvt_f16_f32_e32 v147, v98
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[126:127], v[16:31]
- ; GCN-NEXT: v_fma_f32 v134, s4, v102, -v128
- ; GCN-NEXT: v_mul_f32_e32 v156, 0x3fb8aa3b, v134
- ; GCN-NEXT: buffer_load_dwordx2 v[134:135], v207, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v100, s4, v100, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v133, 0x3fb8aa3b, v100
+ ; GCN-NEXT: v_add_f32_e32 v100, v132, v158
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v114, v162
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[122:123], v[126:127], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v113, v113
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v115, v153
+ ; GCN-NEXT: v_fma_f32 v104, s4, v104, -v148
+ ; GCN-NEXT: v_fma_f32 v101, s4, v101, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v134, 0x3fb8aa3b, v101
+ ; GCN-NEXT: v_pack_b32_f16 v122, v114, v115
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[96:97], v[126:127], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v135, v157
+ ; GCN-NEXT: v_add_f32_e32 v96, v113, v100
+ ; GCN-NEXT: v_fma_f32 v100, s4, v102, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v156
+ ; GCN-NEXT: v_mul_f32_e32 v153, 0x3fb8aa3b, v100
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v100, v154
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[118:119], v[126:127], v[0:15]
+ ; GCN-NEXT: buffer_load_dwordx2 v[118:119], v149, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_exp_f32_e32 v102, v142
- ; GCN-NEXT: buffer_load_dwordx2 v[142:143], v208, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[126:127], v150, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[144:145], v209, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[130:131], v151, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v123, v97, v100
+ ; GCN-NEXT: v_exp_f32_e32 v152, v155
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v132
+ ; GCN-NEXT: v_fma_f32 v101, s4, v103, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[116:117], v[122:123], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v132, v133
+ ; GCN-NEXT: v_mul_f32_e32 v133, 0x3fb8aa3b, v104
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v104, v113
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[126:127], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v99
- ; GCN-NEXT: v_fma_f32 v127, s4, v103, -v128
- ; GCN-NEXT: v_exp_f32_e32 v103, v150
- ; GCN-NEXT: v_fma_f32 v139, s4, v105, -v128
- ; GCN-NEXT: v_pack_b32_f16 v147, v147, v126
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_perm_b32 v152, v135, v131, s5
- ; GCN-NEXT: v_perm_b32 v154, v135, v131, s7
- ; GCN-NEXT: v_fma_f32 v135, s4, v104, -v128
- ; GCN-NEXT: v_perm_b32 v126, v134, v130, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[146:147], v[0:15]
- ; GCN-NEXT: v_perm_b32 v150, v134, v130, s7
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v100
- ; GCN-NEXT: v_exp_f32_e32 v104, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v101
- ; GCN-NEXT: ds_read_b128 v[130:133], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_perm_b32 v127, v144, v142, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[148:149], v[146:147], v[32:47]
- ; GCN-NEXT: v_pack_b32_f16 v148, v134, v135
- ; GCN-NEXT: v_fma_f32 v135, s4, v106, -v128
- ; GCN-NEXT: v_exp_f32_e32 v105, v125
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v102
- ; GCN-NEXT: v_perm_b32 v151, v144, v142, s7
- ; GCN-NEXT: v_perm_b32 v153, v145, v143, s5
- ; GCN-NEXT: v_perm_b32 v155, v145, v143, s7
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[146:147], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v106, v156
- ; GCN-NEXT: v_mul_f32_e32 v156, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v103
- ; GCN-NEXT: v_fma_f32 v136, s4, v107, -v128
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_pack_b32_f16 v149, v134, v135
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[146:147], v[48:63]
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v136
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_exp_f32_e32 v107, v138
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[148:149], v[0:15]
- ; GCN-NEXT: v_fma_f32 v131, s4, v108, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v104
- ; GCN-NEXT: v_exp_f32_e32 v108, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_cvt_f16_f32_e32 v131, v105
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[142:143], v[148:149], v[32:47]
- ; GCN-NEXT: v_fma_f32 v142, s4, v109, -v128
- ; GCN-NEXT: v_exp_f32_e32 v109, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v142
- ; GCN-NEXT: v_pack_b32_f16 v142, v130, v131
- ; GCN-NEXT: v_fma_f32 v131, s4, v110, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v106
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[148:149], v[16:31]
- ; GCN-NEXT: v_mul_f32_e32 v134, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_cvt_f16_f32_e32 v131, v107
- ; GCN-NEXT: v_exp_f32_e32 v110, v156
- ; GCN-NEXT: v_fma_f32 v135, s4, v111, -v128
- ; GCN-NEXT: v_mul_f32_e32 v135, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_pack_b32_f16 v143, v130, v131
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[148:149], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v111, v146
- ; GCN-NEXT: v_fma_f32 v139, s4, v80, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v138, v108
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[142:143], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v80, v129
- ; GCN-NEXT: ds_read_b128 v[130:133], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[146:149], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_cvt_f16_f32_e32 v139, v109
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[144:145], v[142:143], v[32:47]
- ; GCN-NEXT: v_fma_f32 v144, s4, v81, -v128
- ; GCN-NEXT: v_exp_f32_e32 v81, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v144
- ; GCN-NEXT: v_pack_b32_f16 v144, v138, v139
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[142:143], v[16:31]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v110
- ; GCN-NEXT: v_fma_f32 v137, s4, v82, -v128
- ; GCN-NEXT: v_exp_f32_e32 v82, v134
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v111
- ; GCN-NEXT: v_mul_f32_e32 v156, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_fma_f32 v137, s4, v83, -v128
- ; GCN-NEXT: v_mul_f32_e32 v157, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[142:143], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v83, v135
- ; GCN-NEXT: v_pack_b32_f16 v145, v136, v134
- ; GCN-NEXT: ds_read_b128 v[134:137], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[138:141], v197 offset:1728
+ ; GCN-NEXT: v_add_f32_e32 v96, v135, v96
+ ; GCN-NEXT: v_mul_f32_e32 v154, 0x3fb8aa3b, v101
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[124:125], v[122:123], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v113, v134
+ ; GCN-NEXT: ds_read_b128 v[100:103], v139
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v124, v97, v104
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v135
+ ; GCN-NEXT: v_add_f32_e32 v96, v152, v96
+ ; GCN-NEXT: ds_read_b128 v[114:117], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[98:99], v[122:123], v[16:31]
+ ; GCN-NEXT: v_fma_f32 v98, s4, v106, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v135, v153
+ ; GCN-NEXT: v_mul_f32_e32 v149, 0x3fb8aa3b, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v98, v152
+ ; GCN-NEXT: v_add_f32_e32 v96, v132, v96
+ ; GCN-NEXT: v_add_f32_e32 v96, v113, v96
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[120:121], v[122:123], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v120, v154
+ ; GCN-NEXT: v_add_f32_e32 v96, v135, v96
+ ; GCN-NEXT: v_fma_f32 v99, s4, v107, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v125, v97, v98
+ ; GCN-NEXT: v_mul_f32_e32 v121, 0x3fb8aa3b, v99
+ ; GCN-NEXT: v_add_f32_e32 v122, v120, v96
+ ; GCN-NEXT: ds_read_b128 v[96:99], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v105, s4, v105, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v134, 0x3fb8aa3b, v105
+ ; GCN-NEXT: ds_read_b128 v[104:107], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[100:101], v[124:125], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v123, v133
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v100, v132
+ ; GCN-NEXT: v_fma_f32 v101, s4, v108, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v108, v113
+ ; GCN-NEXT: v_mul_f32_e32 v132, 0x3fb8aa3b, v101
+ ; GCN-NEXT: v_add_f32_e32 v101, v123, v122
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[114:115], v[124:125], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v113, v134
+ ; GCN-NEXT: v_pack_b32_f16 v114, v100, v108
+ ; GCN-NEXT: v_fma_f32 v109, s4, v109, -v148
+ ; GCN-NEXT: v_add_f32_e32 v100, v113, v101
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v101, v120
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v113, v113
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[96:97], v[124:125], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v96, v135
+ ; GCN-NEXT: v_exp_f32_e32 v134, v149
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v123
+ ; GCN-NEXT: v_pack_b32_f16 v115, v96, v101
+ ; GCN-NEXT: v_mul_f32_e32 v133, 0x3fb8aa3b, v109
+ ; GCN-NEXT: v_add_f32_e32 v100, v134, v100
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[104:105], v[124:125], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v124, v121
+ ; GCN-NEXT: v_fma_f32 v97, s4, v110, -v148
+ ; GCN-NEXT: v_fma_f32 v80, s4, v80, -v148
+ ; GCN-NEXT: v_add_f32_e32 v125, v124, v100
+ ; GCN-NEXT: v_mul_f32_e32 v97, 0x3fb8aa3b, v97
+ ; GCN-NEXT: v_fma_f32 v104, s4, v111, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[102:103], v[114:115], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v132, v132
+ ; GCN-NEXT: ds_read_b128 v[100:103], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v81, s4, v81, -v148
+ ; GCN-NEXT: v_fma_f32 v82, s4, v82, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v96, 0x3fb8aa3b, v104
+ ; GCN-NEXT: v_perm_b32 v105, v130, v126, s5
+ ; GCN-NEXT: v_perm_b32 v121, v130, v126, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[116:117], v[114:115], v[48:63]
+ ; GCN-NEXT: v_mul_f32_e32 v126, 0x3fb8aa3b, v80
+ ; GCN-NEXT: v_add_f32_e32 v80, v132, v125
+ ; GCN-NEXT: v_exp_f32_e32 v125, v133
+ ; GCN-NEXT: ds_read_b128 v[108:111], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v104, v118, v128, s5
+ ; GCN-NEXT: v_perm_b32 v120, v118, v128, s7
+ ; GCN-NEXT: v_perm_b32 v118, v119, v129, s5
+ ; GCN-NEXT: v_perm_b32 v122, v119, v129, s7
+ ; GCN-NEXT: v_perm_b32 v119, v131, v127, s5
+ ; GCN-NEXT: v_perm_b32 v123, v131, v127, s7
+ ; GCN-NEXT: v_mul_f32_e32 v127, 0x3fb8aa3b, v81
+ ; GCN-NEXT: v_pack_b32_f16 v116, v135, v113
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[98:99], v[114:115], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v134
+ ; GCN-NEXT: v_exp_f32_e32 v113, v97
+ ; GCN-NEXT: v_mul_f32_e32 v128, 0x3fb8aa3b, v82
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v82, v124
+ ; GCN-NEXT: v_add_f32_e32 v80, v125, v80
+ ; GCN-NEXT: v_add_f32_e32 v80, v113, v80
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[106:107], v[114:115], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v114, v96
+ ; GCN-NEXT: v_fma_f32 v83, s4, v83, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v117, v81, v82
+ ; GCN-NEXT: v_mul_f32_e32 v115, 0x3fb8aa3b, v83
+ ; GCN-NEXT: v_add_f32_e32 v106, v114, v80
+ ; GCN-NEXT: ds_read_b128 v[80:83], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[100:101], v[116:117], v[32:47]
+ ; GCN-NEXT: ds_read_b128 v[96:99], v138 offset:1728
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -799,150 +839,164 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[126:127]
+ ; GCN-NEXT: ds_write_b64 v140, v[104:105]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[150:151]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[144:145], v[0:15]
+ ; GCN-NEXT: ds_write_b64 v141, v[120:121]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[152:153]
+ ; GCN-NEXT: ds_write_b64 v142, v[118:119]
+ ; GCN-NEXT: v_exp_f32_e32 v118, v126
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[154:155]
- ; GCN-NEXT: v_fma_f32 v127, s4, v84, -v128
- ; GCN-NEXT: v_exp_f32_e32 v84, v129
- ; GCN-NEXT: v_fma_f32 v130, s4, v85, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v80
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[146:147], v[144:145], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v85, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v130
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx2 v[130:131], v206, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: ds_write_b64 v143, v[122:123]
+ ; GCN-NEXT: v_fma_f32 v84, s4, v84, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[108:109], v[116:117], v[48:63]
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v84
+ ; GCN-NEXT: v_add_f32_e32 v84, v118, v106
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_load_dwordx2 v[106:107], v147, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v127, v81
- ; GCN-NEXT: v_pack_b32_f16 v126, v126, v127
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[144:145], v[16:31]
- ; GCN-NEXT: v_fma_f32 v134, s4, v86, -v128
- ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v134
- ; GCN-NEXT: buffer_load_dwordx2 v[134:135], v203, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[108:109], v145, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[142:143], v204, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_exp_f32_e32 v120, v127
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v100, v132
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[80:81], v[116:117], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v113
+ ; GCN-NEXT: v_add_f32_e32 v80, v120, v84
+ ; GCN-NEXT: v_fma_f32 v84, s4, v86, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v113, v128
+ ; GCN-NEXT: v_mul_f32_e32 v122, 0x3fb8aa3b, v84
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v114
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[96:97], v[116:117], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v96, v115
+ ; GCN-NEXT: buffer_load_dwordx2 v[114:115], v144, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[146:147], v205, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[116:117], v146, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v127, v82
- ; GCN-NEXT: v_exp_f32_e32 v86, v156
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[144:145], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v138, v83
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v101, v125
+ ; GCN-NEXT: v_pack_b32_f16 v105, v81, v84
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v118
+ ; GCN-NEXT: v_fma_f32 v85, s4, v85, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v104, v100, v101
+ ; GCN-NEXT: v_add_f32_e32 v80, v113, v80
+ ; GCN-NEXT: v_mul_f32_e32 v121, 0x3fb8aa3b, v85
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[102:103], v[104:105], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v118, v119
+ ; GCN-NEXT: v_add_f32_e32 v80, v96, v80
+ ; GCN-NEXT: v_fma_f32 v88, s4, v88, -v148
+ ; GCN-NEXT: v_fma_f32 v85, s4, v87, -v148
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_fma_f32 v139, s4, v87, -v128
- ; GCN-NEXT: v_exp_f32_e32 v87, v157
- ; GCN-NEXT: v_pack_b32_f16 v127, v127, v138
- ; GCN-NEXT: v_fma_f32 v138, s4, v89, -v128
- ; GCN-NEXT: v_mul_f32_e32 v139, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[126:127], v[0:15]
- ; GCN-NEXT: ; implicit-def: $sgpr0
- ; GCN-NEXT: v_perm_b32 v154, v135, v131, s5
- ; GCN-NEXT: v_perm_b32 v156, v135, v131, s7
- ; GCN-NEXT: v_fma_f32 v135, s4, v88, -v128
- ; GCN-NEXT: v_perm_b32 v150, v134, v130, s5
- ; GCN-NEXT: v_perm_b32 v152, v134, v130, s7
- ; GCN-NEXT: ds_read_b128 v[130:133], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v84
- ; GCN-NEXT: v_exp_f32_e32 v88, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v85
- ; GCN-NEXT: v_perm_b32 v151, v146, v142, s5
- ; GCN-NEXT: v_perm_b32 v153, v146, v142, s7
- ; GCN-NEXT: v_perm_b32 v155, v147, v143, s5
- ; GCN-NEXT: v_perm_b32 v157, v147, v143, s7
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[148:149], v[126:127], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v89, v125
- ; GCN-NEXT: v_pack_b32_f16 v146, v134, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v86
- ; GCN-NEXT: v_fma_f32 v135, s4, v90, -v128
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v138
- ; GCN-NEXT: v_mul_f32_e32 v148, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[126:127], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v90, v158
- ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v64
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[126:127], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v87
- ; GCN-NEXT: v_fma_f32 v127, s4, v91, -v128
- ; GCN-NEXT: v_exp_f32_e32 v91, v139
- ; GCN-NEXT: v_mul_f32_e32 v127, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_pack_b32_f16 v147, v134, v126
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[146:147], v[0:15]
- ; GCN-NEXT: v_fma_f32 v130, s4, v92, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v88
- ; GCN-NEXT: v_exp_f32_e32 v92, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v130
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v89
- ; GCN-NEXT: v_fma_f32 v131, s4, v93, -v128
- ; GCN-NEXT: v_pack_b32_f16 v130, v126, v130
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[142:143], v[146:147], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v93, v125
- ; GCN-NEXT: v_fma_f32 v126, s4, v94, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v125, v90
- ; GCN-NEXT: v_mul_f32_e32 v143, 0x3fb8aa3b, v126
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v91
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_fma_f32 v131, s4, v95, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[146:147], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v94, v148
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v93
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[146:147], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v95, v127
- ; GCN-NEXT: v_cvt_f16_f32_e32 v127, v92
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_pack_b32_f16 v131, v125, v126
- ; GCN-NEXT: s_nop 1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[130:131], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v125, v129
- ; GCN-NEXT: ds_read_b128 v[132:135], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[146:149], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[144:145], v[130:131], v[32:47]
- ; GCN-NEXT: v_mul_f32_e32 v144, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_fma_f32 v65, s4, v66, -v128
- ; GCN-NEXT: v_exp_f32_e32 v126, v142
- ; GCN-NEXT: v_pack_b32_f16 v142, v127, v64
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v94
- ; GCN-NEXT: v_mul_f32_e32 v145, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v95
- ; GCN-NEXT: v_fma_f32 v66, s4, v67, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[130:131], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v127, v143
- ; GCN-NEXT: v_pack_b32_f16 v143, v64, v65
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[130:131], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v129, v138
- ; GCN-NEXT: v_mul_f32_e32 v141, 0x3fb8aa3b, v66
- ; GCN-NEXT: ds_read_b128 v[64:67], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[136:139], v197 offset:1728
+ ; GCN-NEXT: v_mul_f32_e32 v97, 0x3fb8aa3b, v85
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[110:111], v[104:105], v[48:63]
+ ; GCN-NEXT: v_mul_f32_e32 v110, 0x3fb8aa3b, v88
+ ; GCN-NEXT: v_add_f32_e32 v88, v118, v80
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v80, v120
+ ; GCN-NEXT: v_exp_f32_e32 v111, v121
+ ; GCN-NEXT: ds_read_b128 v[84:87], v139
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[100:103], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[82:83], v[104:105], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v82, v113
+ ; GCN-NEXT: v_exp_f32_e32 v113, v122
+ ; GCN-NEXT: v_pack_b32_f16 v80, v81, v80
+ ; GCN-NEXT: v_add_f32_e32 v81, v111, v88
+ ; GCN-NEXT: v_fma_f32 v89, s4, v89, -v148
+ ; GCN-NEXT: v_add_f32_e32 v88, v113, v81
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[98:99], v[104:105], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v96
+ ; GCN-NEXT: v_exp_f32_e32 v104, v97
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v89
+ ; GCN-NEXT: v_fma_f32 v89, s4, v91, -v148
+ ; GCN-NEXT: v_fma_f32 v83, s4, v90, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v81, v82, v81
+ ; GCN-NEXT: v_mul_f32_e32 v82, 0x3fb8aa3b, v89
+ ; GCN-NEXT: v_add_f32_e32 v105, v104, v88
+ ; GCN-NEXT: ds_read_b128 v[88:91], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[96:99], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[84:85], v[80:81], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v110, v110
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v118
+ ; GCN-NEXT: v_fma_f32 v85, s4, v92, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v92, v111
+ ; GCN-NEXT: v_mul_f32_e32 v83, 0x3fb8aa3b, v83
+ ; GCN-NEXT: v_mul_f32_e32 v118, 0x3fb8aa3b, v85
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[100:101], v[80:81], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v111, v119
+ ; GCN-NEXT: v_add_f32_e32 v85, v110, v105
+ ; GCN-NEXT: v_pack_b32_f16 v92, v84, v92
+ ; GCN-NEXT: v_add_f32_e32 v84, v111, v85
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v85, v113
+ ; GCN-NEXT: v_fma_f32 v93, s4, v93, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[88:89], v[80:81], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v89, v83
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v93
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v110, v110
+ ; GCN-NEXT: v_fma_f32 v88, s4, v94, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v88, 0x3fb8aa3b, v88
+ ; GCN-NEXT: v_perm_b32 v100, v109, v107, s5
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[96:97], v[80:81], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v80, v104
+ ; GCN-NEXT: v_exp_f32_e32 v113, v82
+ ; GCN-NEXT: v_fma_f32 v81, s4, v95, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v93, v85, v80
+ ; GCN-NEXT: v_perm_b32 v104, v109, v107, s7
+ ; GCN-NEXT: v_add_f32_e32 v83, v89, v84
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[86:87], v[92:93], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v118, v118
+ ; GCN-NEXT: v_mul_f32_e32 v120, 0x3fb8aa3b, v81
+ ; GCN-NEXT: ds_read_b128 v[84:87], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v65, s4, v65, -v148
+ ; GCN-NEXT: v_fma_f32 v66, s4, v66, -v148
+ ; GCN-NEXT: v_add_f32_e32 v121, v113, v83
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[102:103], v[92:93], v[48:63]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v102, v111
+ ; GCN-NEXT: v_exp_f32_e32 v107, v119
+ ; GCN-NEXT: ds_read_b128 v[80:83], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v64, s4, v64, -v148
+ ; GCN-NEXT: v_perm_b32 v94, v108, v106, s5
+ ; GCN-NEXT: v_perm_b32 v96, v108, v106, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[90:91], v[92:93], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v109, v88
+ ; GCN-NEXT: v_mul_f32_e32 v108, 0x3fb8aa3b, v65
+ ; GCN-NEXT: v_pack_b32_f16 v102, v110, v102
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v89
+ ; GCN-NEXT: v_mul_f32_e32 v110, 0x3fb8aa3b, v66
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v113
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[98:99], v[92:93], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v92, v120
+ ; GCN-NEXT: v_mul_f32_e32 v106, 0x3fb8aa3b, v64
+ ; GCN-NEXT: v_add_f32_e32 v64, v118, v121
+ ; GCN-NEXT: v_add_f32_e32 v64, v107, v64
+ ; GCN-NEXT: v_add_f32_e32 v64, v109, v64
+ ; GCN-NEXT: v_fma_f32 v67, s4, v67, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v103, v65, v66
+ ; GCN-NEXT: v_mul_f32_e32 v93, 0x3fb8aa3b, v67
+ ; GCN-NEXT: v_add_f32_e32 v98, v92, v64
+ ; GCN-NEXT: ds_read_b128 v[64:67], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v95, v116, v114, s5
+ ; GCN-NEXT: v_perm_b32 v97, v116, v114, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[84:85], v[102:103], v[32:47]
+ ; GCN-NEXT: ds_read_b128 v[88:91], v138 offset:1728
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -950,211 +1004,164 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[150:151]
+ ; GCN-NEXT: ds_write_b64 v140, v[94:95]
+ ; GCN-NEXT: v_exp_f32_e32 v94, v106
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[152:153]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[142:143], v[0:15]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v132, v125
- ; GCN-NEXT: v_exp_f32_e32 v130, v158
+ ; GCN-NEXT: ds_write_b64 v141, v[96:97]
+ ; GCN-NEXT: v_fma_f32 v68, s4, v68, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v95, 0x3fb8aa3b, v68
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[80:81], v[102:103], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v96, v108
+ ; GCN-NEXT: v_add_f32_e32 v68, v94, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v118
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v80, v107
+ ; GCN-NEXT: v_fma_f32 v72, s4, v72, -v148
+ ; GCN-NEXT: v_fma_f32 v69, s4, v69, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[64:65], v[102:103], v[16:31]
+ ; GCN-NEXT: v_add_f32_e32 v64, v96, v68
+ ; GCN-NEXT: v_fma_f32 v68, s4, v70, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v109
+ ; GCN-NEXT: v_mul_f32_e32 v99, 0x3fb8aa3b, v68
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v92
+ ; GCN-NEXT: v_exp_f32_e32 v98, v110
+ ; GCN-NEXT: v_pack_b32_f16 v80, v84, v80
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[88:89], v[102:103], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v88, v93
+ ; GCN-NEXT: v_pack_b32_f16 v81, v65, v68
+ ; GCN-NEXT: v_add_f32_e32 v64, v98, v64
+ ; GCN-NEXT: v_add_f32_e32 v64, v88, v64
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v89, v94
+ ; GCN-NEXT: v_mul_f32_e32 v93, 0x3fb8aa3b, v72
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[86:87], v[80:81], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v92, v95
+ ; GCN-NEXT: v_mul_f32_e32 v97, 0x3fb8aa3b, v69
+ ; GCN-NEXT: v_perm_b32 v101, v117, v115, s5
+ ; GCN-NEXT: v_add_f32_e32 v72, v92, v64
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v96
+ ; GCN-NEXT: v_perm_b32 v105, v117, v115, s7
+ ; GCN-NEXT: v_fma_f32 v69, s4, v71, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[82:83], v[80:81], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v94, v97
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[154:155]
+ ; GCN-NEXT: ds_write_b64 v142, v[100:101]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[156:157]
+ ; GCN-NEXT: ds_write_b64 v143, v[104:105]
+ ; GCN-NEXT: v_mul_f32_e32 v65, 0x3fb8aa3b, v69
+ ; GCN-NEXT: v_fma_f32 v73, s4, v73, -v148
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[146:147], v[142:143], v[32:47]
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v126
- ; GCN-NEXT: v_exp_f32_e32 v131, v144
- ; GCN-NEXT: v_mul_f32_e32 v144, 0x3fb8aa3b, v69
- ; GCN-NEXT: v_fma_f32 v69, s4, v71, -v128
- ; GCN-NEXT: v_pack_b32_f16 v140, v132, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v129
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[64:65], v[142:143], v[16:31]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v127
- ; GCN-NEXT: v_exp_f32_e32 v132, v145
- ; GCN-NEXT: v_fma_f32 v65, s4, v70, -v128
- ; GCN-NEXT: v_mul_f32_e32 v65, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_fma_f32 v145, s4, v73, -v128
- ; GCN-NEXT: v_mul_f32_e32 v147, 0x3fb8aa3b, v145
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[136:137], v[142:143], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v133, v141
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v69
- ; GCN-NEXT: v_pack_b32_f16 v141, v64, v68
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[68:71], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_fma_f32 v143, s4, v72, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v130
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[134:135], v[140:141], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v72, v146
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v143
- ; GCN-NEXT: v_cvt_f16_f32_e32 v143, v131
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_pack_b32_f16 v64, v64, v143
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[148:149], v[140:141], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v73, v144
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[66:67], v[140:141], v[16:31]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v132
- ; GCN-NEXT: v_fma_f32 v67, s4, v74, -v128
- ; GCN-NEXT: v_exp_f32_e32 v74, v65
- ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v133
- ; GCN-NEXT: v_mul_f32_e32 v67, 0x3fb8aa3b, v67
- ; GCN-NEXT: v_pack_b32_f16 v65, v66, v65
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[140:141], v[48:63]
- ; GCN-NEXT: v_fma_f32 v138, s4, v75, -v128
- ; GCN-NEXT: v_exp_f32_e32 v75, v142
- ; GCN-NEXT: v_mul_f32_e32 v148, 0x3fb8aa3b, v138
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v72
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[68:69], v[64:65], v[0:15]
- ; GCN-NEXT: v_fma_f32 v68, s4, v76, -v128
- ; GCN-NEXT: v_exp_f32_e32 v76, v146
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v73
- ; GCN-NEXT: v_fma_f32 v69, s4, v77, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[134:135], v[64:65], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v77, v147
- ; GCN-NEXT: v_pack_b32_f16 v134, v66, v68
- ; GCN-NEXT: v_fma_f32 v68, s4, v78, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v74
- ; GCN-NEXT: v_mul_f32_e32 v147, 0x3fb8aa3b, v69
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[138:139], v[64:65], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v78, v67
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v139, v76
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[142:143], v[64:65], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v75
- ; GCN-NEXT: v_fma_f32 v65, s4, v79, -v128
- ; GCN-NEXT: v_exp_f32_e32 v79, v148
- ; GCN-NEXT: v_mul_f32_e32 v128, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_pack_b32_f16 v135, v66, v64
- ; GCN-NEXT: s_nop 1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[70:71], v[134:135], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v142, v146
- ; GCN-NEXT: ds_read_b128 v[68:71], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[64:67], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[136:137], v[134:135], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v137, v147
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v77
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[140:141], v[134:135], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v138, v138
- ; GCN-NEXT: v_cvt_f16_f32_e32 v140, v78
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[144:145], v[134:135], v[48:63]
- ; GCN-NEXT: s_nop 10
- ; GCN-NEXT: v_exp_f32_e32 v52, v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v50, v137
- ; GCN-NEXT: v_cvt_f16_f32_e32 v51, v142
- ; GCN-NEXT: v_cvt_f16_f32_e32 v54, v138
- ; GCN-NEXT: v_cvt_f16_f32_e32 v53, v52
- ; GCN-NEXT: v_cvt_f16_f32_e32 v49, v79
- ; GCN-NEXT: v_pack_b32_f16 v50, v51, v50
- ; GCN-NEXT: v_pack_b32_f16 v48, v139, v136
- ; GCN-NEXT: v_pack_b32_f16 v51, v54, v53
- ; GCN-NEXT: v_add_f32_e32 v53, 0, v113
- ; GCN-NEXT: v_add_f32_e32 v53, v114, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v115, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v116, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v117, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v118, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v119, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v120, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v121, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v122, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v123, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v124, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v96, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v97, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v98, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v99, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v100, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v101, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v102, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v103, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v104, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v105, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v106, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v107, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v108, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v109, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v110, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v111, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v80, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v81, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v82, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v83, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v84, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v85, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v86, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v87, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v88, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v89, v53
- ; GCN-NEXT: v_pack_b32_f16 v49, v140, v49
- ; GCN-NEXT: v_add_f32_e32 v53, v90, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v91, v53
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[68:69], v[48:49], v[0:15]
- ; GCN-NEXT: v_add_f32_e32 v53, v92, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v93, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v94, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v95, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v125, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v126, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v127, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v129, v53
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[70:71], v[50:51], v[0:15]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[64:65], v[48:49], v[32:47]
- ; GCN-NEXT: s_nop 9
- ; GCN-NEXT: v_add_f32_e32 v0, v130, v53
- ; GCN-NEXT: v_add_f32_e32 v0, v131, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v132, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v133, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v72, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v73, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v74, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v75, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v76, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v77, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v78, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v79, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v142, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v137, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v138, v0
- ; GCN-NEXT: v_add_f32_e32 v4, v52, v0
- ; GCN-NEXT: ds_bpermute_b32 v5, v196, v4
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[0:3], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[0:1], v[48:49], v[16:31]
- ; GCN-NEXT: v_add_f32_e32 v2, v4, v5
- ; GCN-NEXT: ds_bpermute_b32 v3, v196, v2
- ; GCN-NEXT: ; implicit-def: $vgpr4
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[66:67], v[80:81], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v95, v99
+ ; GCN-NEXT: v_pack_b32_f16 v64, v89, v64
+ ; GCN-NEXT: v_mul_f32_e32 v89, 0x3fb8aa3b, v73
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v73, v88
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[12:13]
- ; GCN-NEXT: v_fmac_f32_e32 v0, v4, v112
- ; GCN-NEXT: ds_read_b128 v[0:3], v197 offset:1728
+ ; GCN-NEXT: ds_read_b128 v[68:71], v139
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[90:91], v[80:81], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v88, v65
+ ; GCN-NEXT: v_add_f32_e32 v72, v94, v72
+ ; GCN-NEXT: v_fma_f32 v67, s4, v74, -v148
+ ; GCN-NEXT: v_add_f32_e32 v72, v95, v72
+ ; GCN-NEXT: v_fma_f32 v74, s4, v75, -v148
+ ; GCN-NEXT: ds_read_b128 v[84:87], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v65, v66, v73
+ ; GCN-NEXT: v_mul_f32_e32 v66, 0x3fb8aa3b, v74
+ ; GCN-NEXT: v_add_f32_e32 v90, v88, v72
+ ; GCN-NEXT: ds_read_b128 v[72:75], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[80:83], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[68:69], v[64:65], v[32:47]
+ ; GCN-NEXT: v_mul_f32_e32 v67, 0x3fb8aa3b, v67
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v92
+ ; GCN-NEXT: v_exp_f32_e32 v92, v93
+ ; GCN-NEXT: v_fma_f32 v69, s4, v76, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v69, 0x3fb8aa3b, v69
+ ; GCN-NEXT: v_fma_f32 v77, s4, v77, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[84:85], v[64:65], v[48:63]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v94
+ ; GCN-NEXT: v_exp_f32_e32 v93, v89
+ ; GCN-NEXT: v_add_f32_e32 v76, v92, v90
+ ; GCN-NEXT: v_pack_b32_f16 v84, v68, v84
+ ; GCN-NEXT: v_mul_f32_e32 v94, 0x3fb8aa3b, v77
+ ; GCN-NEXT: v_add_f32_e32 v68, v93, v76
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[72:73], v[64:65], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v72, v95
+ ; GCN-NEXT: v_exp_f32_e32 v95, v67
+ ; GCN-NEXT: v_fma_f32 v73, s4, v78, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v73, 0x3fb8aa3b, v73
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v92, v92
+ ; GCN-NEXT: v_add_f32_e32 v67, v95, v68
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[80:81], v[64:65], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v88
+ ; GCN-NEXT: v_exp_f32_e32 v80, v66
+ ; GCN-NEXT: v_fma_f32 v65, s4, v79, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v85, v72, v64
+ ; GCN-NEXT: v_mul_f32_e32 v81, 0x3fb8aa3b, v65
+ ; GCN-NEXT: v_add_f32_e32 v72, v80, v67
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[70:71], v[84:85], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v96, v69
+ ; GCN-NEXT: ds_read_b128 v[76:79], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[88:91], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[68:71], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[64:67], v138 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_add_f32_e32 v67, v96, v72
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[86:87], v[84:85], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v72, v94
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v93
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v86, v96
+ ; GCN-NEXT: v_add_f32_e32 v67, v72, v67
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v72, v72
+ ; GCN-NEXT: v_pack_b32_f16 v66, v92, v66
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[74:75], v[84:85], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v73, v73
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v74, v95
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[66:67], v[50:51], v[32:47]
+ ; GCN-NEXT: v_add_f32_e32 v75, v73, v67
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v67, v80
+ ; GCN-NEXT: v_exp_f32_e32 v80, v81
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[82:83], v[84:85], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v73, v73
+ ; GCN-NEXT: v_pack_b32_f16 v67, v74, v67
+ ; GCN-NEXT: v_add_f32_e32 v74, v80, v75
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v75, v80
+ ; GCN-NEXT: v_pack_b32_f16 v72, v86, v72
+ ; GCN-NEXT: v_pack_b32_f16 v73, v73, v75
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[76:77], v[66:67], v[32:47]
+ ; GCN-NEXT: ds_bpermute_b32 v76, v136, v74
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_add_f32_e32 v74, v74, v76
+ ; GCN-NEXT: ds_bpermute_b32 v75, v136, v74
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[88:89], v[66:67], v[48:63]
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_cndmask_b32_e64 v74, v75, v74, s[12:13]
+ ; GCN-NEXT: v_fmac_f32_e32 v74, v137, v112
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[68:69], v[66:67], v[16:31]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[78:79], v[72:73], v[32:47]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[90:91], v[72:73], v[48:63]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[70:71], v[72:73], v[16:31]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[64:65], v[66:67], v[0:15]
; GCN-NEXT: s_endpgm
attributes #0 = {"amdgpu-flat-work-group-size"="256,256"}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
index 8dcc737b0c523..f225c42f4515e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
@@ -66,7 +66,6 @@
; GCN-NEXT: ds_read_b128 v[44:47], v49 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[36:37], v[40:41], 0
; GCN-NEXT: ; kill: killed $vgpr1
; GCN-NEXT: ; kill: killed $vgpr0
; GCN-NEXT: v_mul_lo_u32 v76, v76, s6
@@ -78,6 +77,7 @@
; GCN-NEXT: ; implicit-def: $sgpr3
; GCN-NEXT: v_lshl_add_u32 v79, v80, 1, v78
; GCN-NEXT: ; implicit-def: $sgpr0_sgpr1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[36:37], v[40:41], 0
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[44:45], v[40:41], 0
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[38:39], v[42:43], v[16:31]
; GCN-NEXT: ds_read_b128 v[36:39], v51
@@ -148,13 +148,13 @@
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: ds_write_b32 v78, v72
- ; GCN-NEXT: v_mul_f32_e32 v74, s4, v20
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[68:69], v[64:65], v[0:15]
; GCN-NEXT: v_mul_f32_e32 v64, s4, v16
; GCN-NEXT: v_mul_f32_e32 v65, s4, v17
; GCN-NEXT: v_mul_f32_e32 v68, s4, v18
; GCN-NEXT: v_mul_f32_e32 v69, s4, v19
; GCN-NEXT: v_max3_f32 v64, v64, s5, v65
+ ; GCN-NEXT: v_mul_f32_e32 v74, s4, v20
; GCN-NEXT: v_mul_f32_e32 v80, s4, v21
; GCN-NEXT: v_max3_f32 v64, v64, v68, v69
; GCN-NEXT: v_mul_f32_e32 v84, s4, v22
@@ -409,10 +409,10 @@
; GCN-NEXT: v_add_f32_e32 v0, v85, v17
; GCN-NEXT: v_cvt_f16_f32_e32 v17, v26
; GCN-NEXT: v_fma_f32 v12, s4, v12, -v72
- ; GCN-NEXT: v_exp_f32_e32 v8, v8
- ; GCN-NEXT: v_cvt_f16_f32_e32 v28, v67
+ ; GCN-NEXT: v_exp_f32_e32 v28, v8
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v8, v67
; GCN-NEXT: v_fma_f32 v13, s4, v13, -v72
- ; GCN-NEXT: v_exp_f32_e32 v9, v9
+ ; GCN-NEXT: v_exp_f32_e32 v29, v9
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[18:19], v[4:5], v[32:47]
; GCN-NEXT: v_add_f32_e32 v4, v88, v0
; GCN-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v10
@@ -420,78 +420,77 @@
; GCN-NEXT: v_fma_f32 v5, s4, v14, -v72
; GCN-NEXT: v_exp_f32_e32 v10, v0
; GCN-NEXT: v_cvt_f16_f32_e32 v0, v7
- ; GCN-NEXT: v_mul_f32_e32 v11, 0x3fb8aa3b, v11
- ; GCN-NEXT: v_fma_f32 v14, s4, v15, -v72
- ; GCN-NEXT: v_exp_f32_e32 v11, v11
- ; GCN-NEXT: v_pack_b32_f16 v1, v1, v0
- ; GCN-NEXT: v_pack_b32_f16 v0, v17, v28
+ ; GCN-NEXT: v_fma_f32 v9, s4, v15, -v72
; GCN-NEXT: ; implicit-def: $sgpr2
- ; GCN-NEXT: s_nop 1
+ ; GCN-NEXT: v_pack_b32_f16 v1, v1, v0
+ ; GCN-NEXT: v_pack_b32_f16 v0, v17, v8
+ ; GCN-NEXT: v_mul_f32_e32 v8, 0x3fb8aa3b, v11
+ ; GCN-NEXT: v_exp_f32_e32 v11, v8
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[2:3], v[0:1], v[48:63]
; GCN-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v12
; GCN-NEXT: v_add_f32_e32 v2, v30, v4
- ; GCN-NEXT: v_cvt_f16_f32_e32 v4, v8
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v4, v28
; GCN-NEXT: v_exp_f32_e32 v12, v3
; GCN-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v13
- ; GCN-NEXT: v_cvt_f16_f32_e32 v13, v9
- ; GCN-NEXT: v_exp_f32_e32 v15, v3
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v8, v29
+ ; GCN-NEXT: v_exp_f32_e32 v13, v3
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[20:21], v[0:1], v[32:47]
; GCN-NEXT: v_add_f32_e32 v0, v31, v2
; GCN-NEXT: v_add_f32_e32 v0, v22, v0
; GCN-NEXT: v_add_f32_e32 v0, v64, v0
; GCN-NEXT: v_add_f32_e32 v0, v23, v0
; GCN-NEXT: v_add_f32_e32 v0, v25, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v27, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v65, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v68, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v24, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v26, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v67, v0
; GCN-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v5
- ; GCN-NEXT: v_add_f32_e32 v0, v6, v0
+ ; GCN-NEXT: v_add_f32_e32 v0, v27, v0
; GCN-NEXT: v_cvt_f16_f32_e32 v5, v10
- ; GCN-NEXT: v_exp_f32_e32 v17, v1
- ; GCN-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v14
- ; GCN-NEXT: v_add_f32_e32 v0, v7, v0
- ; GCN-NEXT: v_cvt_f16_f32_e32 v14, v11
- ; GCN-NEXT: v_exp_f32_e32 v18, v1
- ; GCN-NEXT: v_add_f32_e32 v6, v8, v0
+ ; GCN-NEXT: v_exp_f32_e32 v14, v1
+ ; GCN-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v9
+ ; GCN-NEXT: v_add_f32_e32 v0, v65, v0
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v9, v11
+ ; GCN-NEXT: v_exp_f32_e32 v15, v1
+ ; GCN-NEXT: v_add_f32_e32 v17, v68, v0
; GCN-NEXT: ds_read_b128 v[0:3], v91
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_add_f32_e32 v6, v9, v6
- ; GCN-NEXT: v_pack_b32_f16 v9, v5, v14
- ; GCN-NEXT: v_pack_b32_f16 v8, v4, v13
- ; GCN-NEXT: v_add_f32_e32 v6, v10, v6
- ; GCN-NEXT: v_cvt_f16_f32_e32 v7, v18
- ; GCN-NEXT: v_cvt_f16_f32_e32 v10, v15
+ ; GCN-NEXT: v_pack_b32_f16 v9, v5, v9
+ ; GCN-NEXT: v_pack_b32_f16 v8, v4, v8
+ ; GCN-NEXT: v_add_f32_e32 v17, v24, v17
+ ; GCN-NEXT: v_add_f32_e32 v17, v26, v17
+ ; GCN-NEXT: v_add_f32_e32 v17, v67, v17
+ ; GCN-NEXT: v_add_f32_e32 v6, v6, v17
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v17, v15
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v18, v13
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[0:1], v[8:9], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v0, v17
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v0, v14
; GCN-NEXT: v_cvt_f16_f32_e32 v4, v12
- ; GCN-NEXT: v_add_f32_e32 v6, v11, v6
- ; GCN-NEXT: v_add_f32_e32 v6, v12, v6
- ; GCN-NEXT: v_add_f32_e32 v1, v15, v6
- ; GCN-NEXT: v_add_f32_e32 v11, v17, v1
- ; GCN-NEXT: v_pack_b32_f16 v1, v0, v7
- ; GCN-NEXT: v_pack_b32_f16 v0, v4, v10
+ ; GCN-NEXT: v_add_f32_e32 v1, v7, v6
+ ; GCN-NEXT: v_add_f32_e32 v19, v28, v1
+ ; GCN-NEXT: v_pack_b32_f16 v1, v0, v17
+ ; GCN-NEXT: v_pack_b32_f16 v0, v4, v18
; GCN-NEXT: ds_read_b128 v[4:7], v91 offset:576
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[4:5], v[8:9], v[32:47]
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[4:5], v[8:9], v[32:47]
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[2:3], v[0:1], v[48:63]
- ; GCN-NEXT: v_add_f32_e32 v2, v18, v11
+ ; GCN-NEXT: v_add_f32_e32 v2, v29, v19
+ ; GCN-NEXT: v_add_f32_e32 v2, v10, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v11, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v12, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v13, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v14, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v15, v2
; GCN-NEXT: ds_bpermute_b32 v3, v66, v2
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[6:7], v[0:1], v[32:47]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_f32_e32 v2, v2, v3
; GCN-NEXT: ds_bpermute_b32 v3, v66, v2
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[6:7], v[0:1], v[32:47]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[0:1]
- ; GCN-NEXT: v_fmac_f32_e32 v2, v4, v16
+ ; GCN-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
+ ; GCN-NEXT: v_fmac_f32_e32 v0, v4, v16
; GCN-NEXT: s_endpgm
attributes #0 = {"amdgpu-flat-work-group-size"="256,256"}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
index 45e5755675d83..7bb9ccf720f3e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
@@ -902,8 +902,7 @@ define amdgpu_kernel void @test_mfma_f32_16x16x8bf16(ptr addrspace(1) %arg) #0 {
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v2, 2
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NEXT: v_mov_b32_e32 v1, 2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
@@ -912,26 +911,27 @@ define amdgpu_kernel void @test_mfma_f32_16x16x8bf16(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
+; GFX90A-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: s_nop 9
+; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x8bf16:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 2
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x8bf16 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: s_nop 10
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x8bf16 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-NEXT: s_nop 9
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index 55c076375fac4..240288ac2a44b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -83,56 +83,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b64_e32 v[0:1], 48
+; AGPR-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; AGPR-NEXT: v_accvgpr_write_b32 a31, s23
+; AGPR-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; AGPR-NEXT: v_accvgpr_write_b32 a30, s22
+; AGPR-NEXT: v_accvgpr_write_b32 a29, s21
+; AGPR-NEXT: v_accvgpr_write_b32 a28, s20
+; AGPR-NEXT: v_accvgpr_write_b32 a27, s19
+; AGPR-NEXT: v_accvgpr_write_b32 a26, s18
+; AGPR-NEXT: v_accvgpr_write_b32 a25, s17
+; AGPR-NEXT: v_accvgpr_write_b32 a24, s16
+; AGPR-NEXT: v_accvgpr_write_b32 a23, s15
+; AGPR-NEXT: v_accvgpr_write_b32 a22, s14
+; AGPR-NEXT: v_accvgpr_write_b32 a21, s13
+; AGPR-NEXT: v_accvgpr_write_b32 a20, s12
+; AGPR-NEXT: v_accvgpr_write_b32 a19, s11
+; AGPR-NEXT: v_accvgpr_write_b32 a18, s10
+; AGPR-NEXT: v_accvgpr_write_b32 a17, s9
+; AGPR-NEXT: v_accvgpr_write_b32 a16, s8
+; AGPR-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[2:5], v[6:9], a[16:31]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], 32
+; AGPR-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-NEXT: v_mov_b32_e32 v9, s17
; AGPR-NEXT: v_mov_b64_e32 v[4:5], 16
-; AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; AGPR-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; AGPR-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
-; AGPR-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
-; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
-; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
-; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
-; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
-; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
-; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
-; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
-; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
-; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
-; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
-; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
-; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
-; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
-; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
-; AGPR-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[8:11], v[12:15], a[0:15]
-; AGPR-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-NEXT: v_mov_b32_e32 v9, s21
-; AGPR-NEXT: v_mov_b32_e32 v10, s22
-; AGPR-NEXT: v_mov_b32_e32 v11, s23
; AGPR-NEXT: v_mov_b64_e32 v[6:7], 0
-; AGPR-NEXT: s_nop 4
-; AGPR-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; AGPR-NEXT: s_nop 6
+; AGPR-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
+; AGPR-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-NEXT: v_mov_b32_e32 v8, s20
+; AGPR-NEXT: v_mov_b32_e32 v9, s21
+; AGPR-NEXT: v_mov_b32_e32 v10, s22
+; AGPR-NEXT: v_mov_b32_e32 v11, s23
; AGPR-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
@@ -149,41 +149,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPR-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; VGPR-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; VGPR-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPR-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPR-NEXT: v_mov_b32_e32 v48, s16
-; VGPR-NEXT: v_mov_b32_e32 v49, s17
-; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[40:43], v[44:47], v[0:15]
-; VGPR-NEXT: v_mov_b32_e32 v50, s18
-; VGPR-NEXT: v_mov_b32_e32 v51, s19
-; VGPR-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPR-NEXT: s_nop 8
-; VGPR-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPR-NEXT: s_nop 1
+; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
+; VGPR-NEXT: s_nop 6
+; VGPR-NEXT: v_mov_b64_e32 v[16:17], 32
+; VGPR-NEXT: v_mov_b64_e32 v[18:19], 16
+; VGPR-NEXT: v_mov_b64_e32 v[20:21], 0
+; VGPR-NEXT: v_mov_b32_e32 v22, s16
+; VGPR-NEXT: s_nop 0
+; VGPR-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
+; VGPR-NEXT: v_mov_b32_e32 v23, s17
+; VGPR-NEXT: v_mov_b32_e32 v24, s18
+; VGPR-NEXT: v_mov_b32_e32 v25, s19
; VGPR-NEXT: v_mov_b32_e32 v0, s20
; VGPR-NEXT: v_mov_b32_e32 v1, s21
; VGPR-NEXT: v_mov_b32_e32 v2, s22
; VGPR-NEXT: v_mov_b32_e32 v3, s23
-; VGPR-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
@@ -192,14 +194,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
-; VGPR-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s12
; VGPR-NEXT: v_mov_b32_e32 v1, s13
; VGPR-NEXT: v_mov_b32_e32 v2, s14
; VGPR-NEXT: v_mov_b32_e32 v3, s15
-; VGPR-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
@@ -280,56 +282,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b64_e32 v[0:1], 48
+; AGPR-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; AGPR-NEXT: v_accvgpr_write_b32 a31, s23
+; AGPR-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; AGPR-NEXT: v_accvgpr_write_b32 a30, s22
+; AGPR-NEXT: v_accvgpr_write_b32 a29, s21
+; AGPR-NEXT: v_accvgpr_write_b32 a28, s20
+; AGPR-NEXT: v_accvgpr_write_b32 a27, s19
+; AGPR-NEXT: v_accvgpr_write_b32 a26, s18
+; AGPR-NEXT: v_accvgpr_write_b32 a25, s17
+; AGPR-NEXT: v_accvgpr_write_b32 a24, s16
+; AGPR-NEXT: v_accvgpr_write_b32 a23, s15
+; AGPR-NEXT: v_accvgpr_write_b32 a22, s14
+; AGPR-NEXT: v_accvgpr_write_b32 a21, s13
+; AGPR-NEXT: v_accvgpr_write_b32 a20, s12
+; AGPR-NEXT: v_accvgpr_write_b32 a19, s11
+; AGPR-NEXT: v_accvgpr_write_b32 a18, s10
+; AGPR-NEXT: v_accvgpr_write_b32 a17, s9
+; AGPR-NEXT: v_accvgpr_write_b32 a16, s8
+; AGPR-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[2:5], v[6:9], a[16:31] cbsz:2 abid:3 blgp:1
; AGPR-NEXT: v_mov_b64_e32 v[2:3], 32
+; AGPR-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-NEXT: v_mov_b32_e32 v9, s17
; AGPR-NEXT: v_mov_b64_e32 v[4:5], 16
-; AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; AGPR-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; AGPR-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
-; AGPR-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
-; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
-; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
-; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
-; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
-; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
-; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
-; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
-; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
-; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
-; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
-; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
-; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
-; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
-; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
-; AGPR-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; AGPR-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-NEXT: v_mov_b32_e32 v9, s21
-; AGPR-NEXT: v_mov_b32_e32 v10, s22
-; AGPR-NEXT: v_mov_b32_e32 v11, s23
; AGPR-NEXT: v_mov_b64_e32 v[6:7], 0
-; AGPR-NEXT: s_nop 4
-; AGPR-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; AGPR-NEXT: s_nop 6
+; AGPR-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
+; AGPR-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-NEXT: v_mov_b32_e32 v8, s20
+; AGPR-NEXT: v_mov_b32_e32 v9, s21
+; AGPR-NEXT: v_mov_b32_e32 v10, s22
+; AGPR-NEXT: v_mov_b32_e32 v11, s23
; AGPR-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
@@ -346,41 +348,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPR-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; VGPR-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; VGPR-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPR-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPR-NEXT: v_mov_b32_e32 v48, s16
-; VGPR-NEXT: v_mov_b32_e32 v49, s17
-; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; VGPR-NEXT: v_mov_b32_e32 v50, s18
-; VGPR-NEXT: v_mov_b32_e32 v51, s19
-; VGPR-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPR-NEXT: s_nop 8
-; VGPR-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPR-NEXT: s_nop 1
+; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPR-NEXT: s_nop 6
+; VGPR-NEXT: v_mov_b64_e32 v[16:17], 32
+; VGPR-NEXT: v_mov_b64_e32 v[18:19], 16
+; VGPR-NEXT: v_mov_b64_e32 v[20:21], 0
+; VGPR-NEXT: v_mov_b32_e32 v22, s16
+; VGPR-NEXT: s_nop 0
+; VGPR-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
+; VGPR-NEXT: v_mov_b32_e32 v23, s17
+; VGPR-NEXT: v_mov_b32_e32 v24, s18
+; VGPR-NEXT: v_mov_b32_e32 v25, s19
; VGPR-NEXT: v_mov_b32_e32 v0, s20
; VGPR-NEXT: v_mov_b32_e32 v1, s21
; VGPR-NEXT: v_mov_b32_e32 v2, s22
; VGPR-NEXT: v_mov_b32_e32 v3, s23
-; VGPR-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
@@ -389,14 +393,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
-; VGPR-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s12
; VGPR-NEXT: v_mov_b32_e32 v1, s13
; VGPR-NEXT: v_mov_b32_e32 v2, s14
; VGPR-NEXT: v_mov_b32_e32 v3, s15
-; VGPR-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 2, i32 3, i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index a1fe463de1c54..1209a1117e17d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -363,41 +363,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v48, s16
-; SDAG-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: s_nop 8
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 32
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 16
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 0
+; SDAG-NEXT: v_mov_b32_e32 v22, s16
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v23, s17
+; SDAG-NEXT: v_mov_b32_e32 v24, s18
+; SDAG-NEXT: v_mov_b32_e32 v25, s19
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -406,14 +408,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -421,50 +423,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -473,56 +476,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT: v_mov_b32_e32 v10, s18
+; HEURRC-NEXT: v_mov_b32_e32 v11, s19
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[2:5], v[6:9], a[16:31]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
+; HEURRC-NEXT: v_mov_b32_e32 v8, s16
+; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15]
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: s_nop 4
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT: s_nop 6
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v8, s20
+; HEURRC-NEXT: v_mov_b32_e32 v9, s21
+; HEURRC-NEXT: v_mov_b32_e32 v10, s22
+; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -539,41 +542,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]
-; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: s_nop 8
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 32
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s16
+; VGPRRC-NEXT: s_nop 0
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v24, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v25, s19
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -582,14 +587,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16:
@@ -729,41 +734,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v48, s16
-; SDAG-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: s_nop 8
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 32
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 16
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 0
+; SDAG-NEXT: v_mov_b32_e32 v22, s16
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v23, s17
+; SDAG-NEXT: v_mov_b32_e32 v24, s18
+; SDAG-NEXT: v_mov_b32_e32 v25, s19
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -772,14 +779,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -787,50 +794,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -839,56 +847,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT: v_mov_b32_e32 v10, s18
+; HEURRC-NEXT: v_mov_b32_e32 v11, s19
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[2:5], v[6:9], a[16:31] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
+; HEURRC-NEXT: v_mov_b32_e32 v8, s16
+; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: s_nop 4
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT: s_nop 6
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v8, s20
+; HEURRC-NEXT: v_mov_b32_e32 v9, s21
+; HEURRC-NEXT: v_mov_b32_e32 v10, s22
+; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -905,41 +913,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: s_nop 8
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 32
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s16
+; VGPRRC-NEXT: s_nop 0
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v24, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v25, s19
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -948,14 +958,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__flags:
@@ -1458,44 +1468,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -1800,44 +1811,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -2856,18 +2868,16 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v36, s24
-; SDAG-NEXT: v_mov_b32_e32 v37, s25
-; SDAG-NEXT: v_mov_b32_e32 v38, s26
-; SDAG-NEXT: v_mov_b32_e32 v39, s27
+; SDAG-NEXT: v_mov_b32_e32 v32, s24
+; SDAG-NEXT: v_mov_b32_e32 v33, s25
+; SDAG-NEXT: v_mov_b32_e32 v34, s26
+; SDAG-NEXT: v_mov_b32_e32 v35, s27
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v40, s28
-; SDAG-NEXT: v_mov_b32_e32 v41, s29
-; SDAG-NEXT: v_mov_b32_e32 v42, s30
-; SDAG-NEXT: v_mov_b32_e32 v43, s31
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -2876,11 +2886,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16
@@ -2895,14 +2908,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b32_e32 v1, s17
; SDAG-NEXT: v_mov_b32_e32 v2, s18
; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s8
@@ -2924,50 +2937,51 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -2975,18 +2989,16 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
-; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v4, s24
-; HEURRC-NEXT: v_mov_b32_e32 v5, s25
-; HEURRC-NEXT: v_mov_b32_e32 v6, s26
-; HEURRC-NEXT: v_mov_b32_e32 v7, s27
+; HEURRC-NEXT: v_mov_b32_e32 v0, s24
+; HEURRC-NEXT: v_mov_b32_e32 v1, s25
+; HEURRC-NEXT: v_mov_b32_e32 v2, s26
+; HEURRC-NEXT: v_mov_b32_e32 v3, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
-; HEURRC-NEXT: v_mov_b32_e32 v8, s28
-; HEURRC-NEXT: v_mov_b32_e32 v9, s29
-; HEURRC-NEXT: v_mov_b32_e32 v10, s30
-; HEURRC-NEXT: v_mov_b32_e32 v11, s31
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
@@ -3002,12 +3014,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
-; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
-; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
-; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
; HEURRC-NEXT: s_nop 5
@@ -3046,18 +3059,16 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3066,11 +3077,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
-; VGPRRC-NEXT: s_nop 11
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 48
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 32
+; VGPRRC-NEXT: s_nop 2
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 16
@@ -3085,14 +3099,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
@@ -3257,18 +3271,16 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v36, s24
-; SDAG-NEXT: v_mov_b32_e32 v37, s25
-; SDAG-NEXT: v_mov_b32_e32 v38, s26
-; SDAG-NEXT: v_mov_b32_e32 v39, s27
+; SDAG-NEXT: v_mov_b32_e32 v32, s24
+; SDAG-NEXT: v_mov_b32_e32 v33, s25
+; SDAG-NEXT: v_mov_b32_e32 v34, s26
+; SDAG-NEXT: v_mov_b32_e32 v35, s27
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v40, s28
-; SDAG-NEXT: v_mov_b32_e32 v41, s29
-; SDAG-NEXT: v_mov_b32_e32 v42, s30
-; SDAG-NEXT: v_mov_b32_e32 v43, s31
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3277,11 +3289,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16
@@ -3296,14 +3311,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b32_e32 v1, s17
; SDAG-NEXT: v_mov_b32_e32 v2, s18
; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s8
@@ -3325,50 +3340,51 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -3376,18 +3392,16 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
-; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v4, s24
-; HEURRC-NEXT: v_mov_b32_e32 v5, s25
-; HEURRC-NEXT: v_mov_b32_e32 v6, s26
-; HEURRC-NEXT: v_mov_b32_e32 v7, s27
+; HEURRC-NEXT: v_mov_b32_e32 v0, s24
+; HEURRC-NEXT: v_mov_b32_e32 v1, s25
+; HEURRC-NEXT: v_mov_b32_e32 v2, s26
+; HEURRC-NEXT: v_mov_b32_e32 v3, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
-; HEURRC-NEXT: v_mov_b32_e32 v8, s28
-; HEURRC-NEXT: v_mov_b32_e32 v9, s29
-; HEURRC-NEXT: v_mov_b32_e32 v10, s30
-; HEURRC-NEXT: v_mov_b32_e32 v11, s31
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
@@ -3403,12 +3417,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
-; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
-; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
-; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
; HEURRC-NEXT: s_nop 5
@@ -3447,18 +3462,16 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3467,11 +3480,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
-; VGPRRC-NEXT: s_nop 11
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 48
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 32
+; VGPRRC-NEXT: s_nop 2
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 16
@@ -3486,14 +3502,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
@@ -4029,44 +4045,45 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -4406,44 +4423,45 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -4716,20 +4734,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v20, s24
-; SDAG-NEXT: v_mov_b32_e32 v21, s25
-; SDAG-NEXT: v_mov_b32_e32 v22, s26
-; SDAG-NEXT: v_mov_b32_e32 v23, s27
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s24
+; SDAG-NEXT: v_mov_b32_e32 v17, s25
+; SDAG-NEXT: v_mov_b32_e32 v18, s26
+; SDAG-NEXT: v_mov_b32_e32 v19, s27
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v20, s28
+; SDAG-NEXT: v_mov_b32_e32 v21, s29
+; SDAG-NEXT: v_mov_b32_e32 v22, s30
+; SDAG-NEXT: v_mov_b32_e32 v23, s31
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4777,20 +4794,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v20, s24
-; HEURRC-NEXT: v_mov_b32_e32 v21, s25
-; HEURRC-NEXT: v_mov_b32_e32 v22, s26
-; HEURRC-NEXT: v_mov_b32_e32 v23, s27
+; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v16, s24
+; HEURRC-NEXT: v_mov_b32_e32 v17, s25
+; HEURRC-NEXT: v_mov_b32_e32 v18, s26
+; HEURRC-NEXT: v_mov_b32_e32 v19, s27
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b32_e32 v20, s28
+; HEURRC-NEXT: v_mov_b32_e32 v21, s29
+; HEURRC-NEXT: v_mov_b32_e32 v22, s30
+; HEURRC-NEXT: v_mov_b32_e32 v23, s31
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4810,20 +4826,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
+; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b32_e32 v20, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v21, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4922,20 +4937,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v20, s24
-; SDAG-NEXT: v_mov_b32_e32 v21, s25
-; SDAG-NEXT: v_mov_b32_e32 v22, s26
-; SDAG-NEXT: v_mov_b32_e32 v23, s27
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s24
+; SDAG-NEXT: v_mov_b32_e32 v17, s25
+; SDAG-NEXT: v_mov_b32_e32 v18, s26
+; SDAG-NEXT: v_mov_b32_e32 v19, s27
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v20, s28
+; SDAG-NEXT: v_mov_b32_e32 v21, s29
+; SDAG-NEXT: v_mov_b32_e32 v22, s30
+; SDAG-NEXT: v_mov_b32_e32 v23, s31
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4983,20 +4997,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v20, s24
-; HEURRC-NEXT: v_mov_b32_e32 v21, s25
-; HEURRC-NEXT: v_mov_b32_e32 v22, s26
-; HEURRC-NEXT: v_mov_b32_e32 v23, s27
+; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v16, s24
+; HEURRC-NEXT: v_mov_b32_e32 v17, s25
+; HEURRC-NEXT: v_mov_b32_e32 v18, s26
+; HEURRC-NEXT: v_mov_b32_e32 v19, s27
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b32_e32 v20, s28
+; HEURRC-NEXT: v_mov_b32_e32 v21, s29
+; HEURRC-NEXT: v_mov_b32_e32 v22, s30
+; HEURRC-NEXT: v_mov_b32_e32 v23, s31
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -5016,20 +5029,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
+; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b32_e32 v20, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v21, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 4048f446371ef..27738ef1865fa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -246,8 +246,7 @@ define amdgpu_kernel void @test_mfma_i32_16x16x16i8(ptr addrspace(1) %arg) #0 {
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v2, 2
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NEXT: v_mov_b32_e32 v1, 2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
@@ -256,26 +255,27 @@ define amdgpu_kernel void @test_mfma_i32_16x16x16i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
+; GFX90A-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: s_nop 9
+; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_i32_16x16x16i8:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 2
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: s_nop 10
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-VGPR-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-NEXT: s_nop 9
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 8152b5e38477c..5293ba1754700 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -1392,51 +1392,51 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 2.0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: s_nop 9
+; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: test_mfma_f32_16x16x4f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 2.0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-NEXT: s_nop 9
-; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: s_nop 8
+; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 2.0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT: s_nop 1
-; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-NEXT: s_nop 9
-; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-NEXT: s_nop 8
+; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
@@ -2817,7 +2817,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; NOLIT-SRCC-LABEL: test_mfma_f32_16x16x16f16:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
@@ -2825,16 +2824,17 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s4
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s8
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s5
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s9
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s9
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v5
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v7, s10
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s10
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s11
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v6
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v7
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v6
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v5
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s6
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
@@ -2879,7 +2879,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; LIT-SRCC-LABEL: test_mfma_f32_16x16x16f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; LIT-SRCC-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
@@ -2887,16 +2886,17 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s4
; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s8
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s5
-; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s9
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s9
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v5
-; LIT-SRCC-NEXT: v_mov_b32_e32 v7, s10
+; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s10
; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s11
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v6
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v7
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v6
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v5
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s6
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; LIT-SRCC-NEXT: s_nop 0
; LIT-SRCC-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
@@ -2910,20 +2910,20 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX90A-LABEL: test_mfma_f32_16x16x16f16:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v6, s4
-; GFX90A-NEXT: v_mov_b32_e32 v7, s5
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: v_mov_b32_e32 v5, s5
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v8, s6
-; GFX90A-NEXT: v_mov_b32_e32 v9, s7
+; GFX90A-NEXT: v_mov_b32_e32 v6, s6
+; GFX90A-NEXT: v_mov_b32_e32 v7, s7
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[6:7], v[8:9], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
+; GFX90A-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: s_nop 9
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index 20f6f5021d5ef..eb5290a6710ca 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -1158,7 +1158,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
@@ -1182,12 +1181,13 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v21, s12
-; SDAG-NEXT: v_mov_b32_e32 v22, s13
+; SDAG-NEXT: v_mov_b32_e32 v20, s12
+; SDAG-NEXT: v_mov_b32_e32 v21, s13
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -1223,10 +1223,9 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v21, -2
-; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v20, -2
+; SDAG-NEXT: v_mov_b32_e32 v21, 0x41
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
@@ -1247,9 +1246,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v15, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v20 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
@@ -1286,10 +1286,9 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v21, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v20, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0x41
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
@@ -1310,9 +1309,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v15, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v20 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
@@ -1349,10 +1349,9 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v21, -2
-; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v20, -2
+; SDAG-NEXT: v_mov_b32_e32 v21, 1.0
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
@@ -1373,9 +1372,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v15, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v20 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
@@ -1412,10 +1412,9 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v21, 0.15915494
-; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0.15915494
+; SDAG-NEXT: v_mov_b32_e32 v21, 1.0
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
@@ -1436,9 +1435,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v15, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v20 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
diff --git a/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir b/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir
index 4d1a663aace42..54a4f89eae268 100644
--- a/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir
+++ b/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir
@@ -221,22 +221,21 @@ body: |
;
; gfx908-PAD25-LABEL: name: mfma_padding_8_pass
; gfx908-PAD25: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD25-NEXT: S_NOP 1
; gfx908-PAD25-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD50-LABEL: name: mfma_padding_8_pass
; gfx908-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD50-NEXT: S_NOP 3
+ ; gfx908-PAD50-NEXT: S_NOP 0
; gfx908-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD75-LABEL: name: mfma_padding_8_pass
; gfx908-PAD75: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD75-NEXT: S_NOP 5
+ ; gfx908-PAD75-NEXT: S_NOP 0
; gfx908-PAD75-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD100-LABEL: name: mfma_padding_8_pass
; gfx908-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD100-NEXT: S_NOP 7
+ ; gfx908-PAD100-NEXT: S_NOP 1
; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-DEFAULT-LABEL: name: mfma_padding_8_pass
@@ -245,12 +244,12 @@ body: |
;
; gfx90a-PAD50-LABEL: name: mfma_padding_8_pass
; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx90a-PAD50-NEXT: S_NOP 3
+ ; gfx90a-PAD50-NEXT: S_NOP 0
; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-PAD100-LABEL: name: mfma_padding_8_pass
; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx90a-PAD100-NEXT: S_NOP 7
+ ; gfx90a-PAD100-NEXT: S_NOP 1
; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-DEFAULT-LABEL: name: mfma_padding_8_pass
@@ -259,12 +258,12 @@ body: |
;
; gfx942-PAD50-LABEL: name: mfma_padding_8_pass
; gfx942-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx942-PAD50-NEXT: S_NOP 3
+ ; gfx942-PAD50-NEXT: S_NOP 0
; gfx942-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-PAD100-LABEL: name: mfma_padding_8_pass
; gfx942-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx942-PAD100-NEXT: S_NOP 7
+ ; gfx942-PAD100-NEXT: S_NOP 1
; gfx942-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
@@ -290,21 +289,18 @@ body: |
; gfx908-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx908-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx908-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx908-PAD50-NEXT: S_NOP 1
; gfx908-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD75-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx908-PAD75: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx908-PAD75-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx908-PAD75-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx908-PAD75-NEXT: S_NOP 3
; gfx908-PAD75-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx908-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx908-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx908-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx908-PAD100-NEXT: S_NOP 5
; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-DEFAULT-LABEL: name: mfma_padding_8_pass_2_intervening_valu
@@ -317,14 +313,12 @@ body: |
; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx90a-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx90a-PAD50-NEXT: S_NOP 1
; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx90a-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx90a-PAD100-NEXT: S_NOP 5
; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-DEFAULT-LABEL: name: mfma_padding_8_pass_2_intervening_valu
@@ -337,14 +331,12 @@ body: |
; gfx942-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx942-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx942-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx942-PAD50-NEXT: S_NOP 1
; gfx942-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx942-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx942-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx942-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx942-PAD100-NEXT: S_NOP 5
; gfx942-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$vgpr2 = V_MOV_B32_e32 1, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index 401ad6489bf6e..28143ca51cc8f 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -45,7 +45,7 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
; GFX908-NEXT: s_mov_b32 s2, 0
-; GFX908-NEXT: s_mov_b32 s3, 0
+; GFX908-NEXT: s_mov_b32 s6, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_cmp_lg_u32 s0, 0
; GFX908-NEXT: s_cselect_b64 s[0:1], -1, 0
@@ -54,28 +54,27 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX908-NEXT: s_branch .LBB0_2
; GFX908-NEXT: .LBB0_1: ; %bb2
; GFX908-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; GFX908-NEXT: s_nop 6
+; GFX908-NEXT: s_nop 5
; GFX908-NEXT: v_accvgpr_read_b32 v3, a2
-; GFX908-NEXT: s_or_b32 s4, s3, 1
-; GFX908-NEXT: s_ashr_i32 s5, s3, 31
; GFX908-NEXT: s_mov_b32 s3, s2
; GFX908-NEXT: v_mov_b32_e32 v1, s2
; GFX908-NEXT: v_mov_b32_e32 v2, s3
; GFX908-NEXT: v_accvgpr_write_b32 a0, v3
; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
; GFX908-NEXT: v_accvgpr_read_b32 v3, a1
-; GFX908-NEXT: s_and_b32 s3, s5, s4
+; GFX908-NEXT: s_or_b32 s3, s6, 1
; GFX908-NEXT: v_accvgpr_write_b32 a2, v4
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: s_ashr_i32 s4, s6, 31
; GFX908-NEXT: v_mfma_f32_16x16x16f16 a[2:5], v[1:2], v[1:2], a[0:3]
+; GFX908-NEXT: s_and_b32 s6, s4, s3
; GFX908-NEXT: s_cbranch_execz .LBB0_4
; GFX908-NEXT: .LBB0_2: ; %bb
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX908-NEXT: s_cbranch_vccz .LBB0_1
; GFX908-NEXT: ; %bb.3:
-; GFX908-NEXT: ; implicit-def: $sgpr3
+; GFX908-NEXT: ; implicit-def: $sgpr6
; GFX908-NEXT: ; implicit-def: $agpr2
; GFX908-NEXT: .LBB0_4: ; %common.ret
; GFX908-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir b/llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir
new file mode 100644
index 0000000000000..8eca07927bea7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir
@@ -0,0 +1,31 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx950 -run-pass=machine-scheduler --misched-prera-direction=topdown -verify-machineinstrs %s -o - -debug-only=machine-scheduler 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# BUFFER_LOAD (multi-cycle) -> 8-pass MFMA -> VALU consuming load data.
+# VALU stalls for operand readiness; no spurious HWVALU hazard from MFMA.
+
+# CHECK: ScheduleDAGMILive::schedule starting
+# CHECK: Scheduling SU({{[0-9]+}}) {{.*}} BUFFER_LOAD_DWORDX4
+# CHECK: Scheduling SU({{[0-9]+}}) {{.*}} V_MFMA_F32_32X32X16_F16
+# CHECK: HWVALU +2x1u
+# CHECK: HWXDL +8x1u
+# CHECK: Scheduling SU({{[0-9]+}}) {{.*}} V_ADD_F32
+# CHECK: *** Stall until: [[STALL:[0-9]+]]
+# CHECK-NOT: hazard: SU({{[0-9]+}}) HWVALU[0]={{[0-9]+}}c, is later than CurrCycle = {{[0-9]+}}c
+
+---
+name: mfma_valu_after_load
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+ %0:sgpr_128 = IMPLICIT_DEF
+ %1:vreg_128_align2 = IMPLICIT_DEF
+ %2:vreg_128_align2 = IMPLICIT_DEF
+ %3:vreg_512_align2 = IMPLICIT_DEF
+ %14:vgpr_32 = IMPLICIT_DEF
+ %4:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET %0, 0, 0, 0, 0, implicit $exec
+ %3:vreg_512_align2 = V_MFMA_F32_32X32X16_F16_mac_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+ %5:vgpr_32 = nofpexcept V_ADD_F32_e32 %4.sub0, %14, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %3, implicit %5
+...
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir b/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir
index 982f1b8d89eb7..c51c7964c48be 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir
@@ -3,12 +3,15 @@
# Check that cycle counts are consistent with hazards.
-# CHECK: Cycle: 3 TopQ.A
-# CHECK: hazard: SU(6) HWXDL[0]=9c, is later than CurrCycle = 3c
-# CHECK-NOT: Cycle: 9 TopQ.A
-# CHECK: Cycle: 83 TopQ.A
# CHECK: Checking pending node SU(6)
-# CHECK: Move SU(6) into Available Q
+# CHECK: hazard: SU(6) HWVALU[0]=3c, is later than CurrCycle = 2c
+# CHECK: Scheduling SU(7) %5.sub0:vreg_128_align2 = V_ADD_U32_e32 %4.sub0:vreg_128, %1:vgpr_32, implicit $exec
+# CHECK: *** Stall until: 80
+# CHECK: HWVALU +1x1u
+# CHECK: Resource conflict: HWVALU[0] reserved until @3
+# CHECK: Scheduling SU(6) %3:areg_512 = V_MFMA_F32_16X16X1F32_mac_e64 %2:vgpr_32, %1:vgpr_32, %3:areg_512(tied-def 0), 0, 0, 0, implicit $mode, implicit $exec
+# CHECK: HWVALU +2x1u
+# CHECK: HWXDL +8x1u
---
name: pending_queue_ready_cycle
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
index c233b89369ce2..345a6527928b6 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
@@ -31,8 +31,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_pk_mul_unpacking_f32
@@ -53,8 +52,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_pk_mul_unpacking_f32
@@ -124,8 +122,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_op_sel_selection_unpacking_f32
@@ -146,8 +143,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_op_sel_selection_unpacking_f32
@@ -217,8 +213,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_op_sel_hi_selection_unpacking_f32
@@ -239,8 +234,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_op_sel_hi_selection_unpacking_f32
@@ -322,12 +316,9 @@ body: |
; GFX950-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr18, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr19, implicit $exec, implicit $exec
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_BF8_BF8_e64 killed $vgpr4_vgpr5, killed $vgpr2_vgpr3, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 1, 2, 3, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr2_vgpr3 = nofpexcept V_PK_ADD_F32 8, killed $sgpr2_sgpr3, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr6_vgpr7 = nofpexcept V_PK_ADD_F32 8, killed $sgpr6_sgpr7, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_ADD_F32 8, killed $sgpr4_sgpr5, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
@@ -361,12 +352,9 @@ body: |
; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr18, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr19, implicit $exec, implicit $exec
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_BF8_BF8_e64 killed $vgpr4_vgpr5, killed $vgpr2_vgpr3, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 1, 2, 3, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr2_vgpr3 = nofpexcept V_PK_ADD_F32 8, killed $sgpr2_sgpr3, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr6_vgpr7 = nofpexcept V_PK_ADD_F32 8, killed $sgpr6_sgpr7, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_ADD_F32 8, killed $sgpr4_sgpr5, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -465,8 +453,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 8, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_pk_fma_unpacking_f32
@@ -485,8 +472,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 8, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_pk_fma_unpacking_f32
@@ -550,8 +536,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_MUL_F32 8, $sgpr30_sgpr31, 12, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_FMA_F32 8, killed $sgpr30_sgpr31, 8, $vgpr4_vgpr5, 0, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
@@ -571,8 +556,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_MUL_F32 8, $sgpr30_sgpr31, 12, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_FMA_F32 8, killed $sgpr30_sgpr31, 8, $vgpr4_vgpr5, 0, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -740,8 +724,7 @@ body: |
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 $sgpr15, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr6 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr7 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 8, killed $vgpr6_vgpr7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_opsel_register_is_correctly_marked_as_killed
@@ -762,8 +745,7 @@ body: |
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 $sgpr15, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr6 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr7 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 8, killed $vgpr6_vgpr7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_opsel_register_is_correctly_marked_as_killed
@@ -1011,8 +993,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, 1065353216, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_unpacking_with_imm_input
@@ -1033,8 +1014,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, 1065353216, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_unpacking_with_imm_input
@@ -1104,8 +1084,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 11, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_neg_lo_hi_post_unpacking
@@ -1126,8 +1105,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 11, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_neg_lo_hi_post_unpacking
diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s
index 880946e937eb8..e4c49c094e57a 100644
--- a/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s
+++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s
@@ -143,8 +143,8 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK: [6] - HWXDL
# CHECK: [0] [1] [2] [3] [4] [5] [6] Instructions:
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3]
@@ -153,54 +153,54 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - 8.00 - - v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 v[0:31], v0, v1, v[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 v[0:31], v0, v1, v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 a[0:31], v0, v1, a[34:65]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 v[10:13], a[2:3], v[4:7], v0 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 a[10:13], v[2:3], a[4:7], v1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 v[10:13], a[2:3], v[4:7], v4 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 a[10:13], v[2:3], a[4:7], v5
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 v[10:13], a[2:3], v[4:7], v8 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 a[10:13], v[2:3], a[4:7], v9
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
@@ -209,15 +209,15 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s
index 33665b655086c..050e416b23256 100644
--- a/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s
+++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s
@@ -189,63 +189,63 @@ v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK: [0] [1] [2] [3] [4] [5] [6] Instructions:
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[0:3], a[4:7]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_f16 v[0:15], v[0:3], v[0:3], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_f16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf16 v[0:15], v[0:3], v[0:3], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[0:3], a[4:7]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x32_i8 v[0:15], v[0:3], v[0:3], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x32_i8 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[0:3], a[4:7]
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[0:3], a[4:7]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_f16 v[0:15], v[0:3], v[0:3], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_f16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf16 v[0:15], v[0:3], v[0:3], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[0:3], a[4:7]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x32_i8 v[0:15], v[0:3], v[0:3], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x32_i8 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[0:3], a[4:7]
# CHECK-NEXT: - - - - 1.00 - - v_mfma_ld_scale_b32 v0, v0
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3]
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:3
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:7], v[0:3] blgp:4
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] cbsz:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:3
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:7], v[4:11], v[0:3] cbsz:4
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2 blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] cbsz:1 blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:3
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:7], v[0:15] blgp:4
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] cbsz:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:3
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:7], v[4:11], v[0:15] cbsz:4
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3]
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:3
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:7], v[0:3] blgp:4
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] cbsz:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:3
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:7], v[4:11], v[0:3] cbsz:4
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2 blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] cbsz:1 blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:7], v[0:15] blgp:4
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] cbsz:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:7], v[4:11], v[0:15] cbsz:4
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3], v5, v5 op_sel_hi:[0,0,0]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3], v5, v5 op_sel_hi:[0,0,0] blgp:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:9], v[0:3], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:9], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:9], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:2
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_f16 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_f16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_f16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf16 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x128_i8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x64_i8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x64_i8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_bf8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_bf8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_fp8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_fp8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_bf8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_bf8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_fp8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_fp8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3]
@@ -254,54 +254,54 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - 16.00 - - v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 v[0:31], v0, v1, v[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 v[0:31], v0, v1, v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 a[0:31], v0, v1, a[34:65]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 v[10:13], a[2:3], v[4:7], v0 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 a[10:13], v[2:3], a[4:7], v1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 v[10:13], a[2:3], v[4:7], v4 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 a[10:13], v[2:3], a[4:7], v5
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 v[10:13], a[2:3], v[4:7], v8 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 a[10:13], v[2:3], a[4:7], v9
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
@@ -310,15 +310,15 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
More information about the llvm-commits
mailing list