[llvm] [AMDGPU] Model 2-pass HWVALU blocking for 8-pass XDL MFMA (PR #184084)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 20:48:25 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/184084
>From 8bb61e6e900d40f3302f0c63b94a4f7f007b0456 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Wed, 27 May 2026 10:12:35 -0500
Subject: [PATCH] [AMDGPU] Model 2-pass HWVALU blocking for 8-pass XDL MFMA
Reserve HWVALU for two cycles on Write8PassMAI so the machine
scheduler reflects that VALU cannot co-execute with MFMA during
its first two passes. Also record multi-cycle buffered reservations
in SchedBoundary so structural hazards apply to dual-resource writes.
Add schedule-mfma-valu-after-load.mir and refresh affected tests.
---
llvm/lib/CodeGen/MachineScheduler.cpp | 5 +-
llvm/lib/Target/AMDGPU/SISchedule.td | 6 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 219 +-
.../AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir | 1805 ++++++------
.../AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir | 89 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll | 20 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 692 ++---
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 1284 +++++----
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll | 316 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 2563 +----------------
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 2140 +-------------
.../AMDGPU/neighboring-mfma-padding.mir | 22 +-
.../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll | 25 +-
.../AMDGPU/schedule-mfma-valu-after-load.mir | 31 +
.../CodeGen/AMDGPU/schedule-pending-queue.mir | 13 +-
...ck-non-coissue-insts-post-ra-scheduler.mir | 66 +-
llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s | 60 +-
llvm/test/tools/llvm-mca/AMDGPU/gfx950.s | 148 +-
18 files changed, 2569 insertions(+), 6935 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 9cebfa35af752..ff249fca5de99 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -3010,7 +3010,10 @@ void SchedBoundary::bumpNode(SUnit *SU) {
PI = SchedModel->getWriteProcResBegin(SC),
PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
unsigned PIdx = PI->ProcResourceIdx;
- if (SchedModel->getResourceBufferSize(PIdx) == 0) {
+ // Track unbuffered resources, and buffered resources held for multiple
+ // cycles.
+ if (SchedModel->getResourceBufferSize(PIdx) == 0 ||
+ PI->ReleaseAtCycle > PI->AcquireAtCycle) {
if (SchedModel && SchedModel->enableIntervals()) {
unsigned ReservedUntil, InstanceIdx;
diff --git a/llvm/lib/Target/AMDGPU/SISchedule.td b/llvm/lib/Target/AMDGPU/SISchedule.td
index 8dbe4384eb6f1..716b5fccbe4a7 100644
--- a/llvm/lib/Target/AMDGPU/SISchedule.td
+++ b/llvm/lib/Target/AMDGPU/SISchedule.td
@@ -199,8 +199,10 @@ multiclass SICommonWriteRes {
def : HWWriteRes<Write2PassMAI, [HWXDL], 2>;
let ReleaseAtCycles = [4] in
def : HWWriteRes<Write4PassMAI, [HWXDL], 4>;
- let ReleaseAtCycles = [8] in
- def : HWWriteRes<Write8PassMAI, [HWXDL], 8>;
+ // 8-pass XDL MFMA blocks HWVALU for the first two passes; VALU may
+ // co-execute with MFMA starting at pass 3.
+ let ReleaseAtCycles = [2, 8] in
+ def : HWWriteRes<Write8PassMAI, [HWVALU, HWXDL], 8>;
let ReleaseAtCycles = [16] in
def : HWWriteRes<Write16PassMAI, [HWXDL], 16>;
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 41c3e0a07f5f5..bd4d14ca95c83 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -370,76 +370,151 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX908-LABEL: v32_asm_def_use:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v33, v1
-; GFX908-NEXT: v_mov_b32_e32 v34, v0
+; GFX908-NEXT: v_mov_b32_e32 v32, v1
+; GFX908-NEXT: v_mov_b32_e32 v33, v0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def v[0:31] a[0:15]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a15
-; GFX908-NEXT: ;;#ASMSTART
-; GFX908-NEXT: ; def v32
-; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a14
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a14
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a13
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a13
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a12
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a12
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a11
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a11
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a10
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a9
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a8
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a7
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a6
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a5
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a4
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a3
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v35
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v39
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v35
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX908-NEXT: s_nop 9
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a0 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a3 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a4 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a5 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a6 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a7 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a8 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a9 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX908-NEXT: ;;#ASMSTART
+; GFX908-NEXT: ; def v32
+; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a1
-; GFX908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_write_b32 a32, v35
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a1
+; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v39
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v39 ; Reload Reuse
+; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; copy
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: s_nop 7
-; GFX908-NEXT: v_accvgpr_read_b32 v35, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v39, a2
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v35
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v39
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; use a3 v[0:31]
; GFX908-NEXT: ;;#ASMEND
@@ -451,9 +526,8 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-LABEL: v32_asm_def_use:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_read_b32 v35, a32 ; Reload Reuse
-; GFX90A-NEXT: v_mov_b32_e32 v34, v0
-; GFX90A-NEXT: v_mov_b32_e32 v33, v1
+; GFX90A-NEXT: v_mov_b32_e32 v33, v0
+; GFX90A-NEXT: v_mov_b32_e32 v32, v1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:31] a[0:15]
; GFX90A-NEXT: ;;#ASMEND
@@ -473,20 +547,53 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
; GFX90A-NEXT: v_accvgpr_mov_b32 a18, a2
; GFX90A-NEXT: v_accvgpr_mov_b32 a17, a1
; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a0
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v33, v32, a[16:31]
+; GFX90A-NEXT: s_nop 10
+; GFX90A-NEXT: buffer_store_dword a0, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX90A-NEXT: s_nop 0
+; GFX90A-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX90A-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v32
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_accvgpr_read_b32 v39, a10 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v38, a11 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v37, a12 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v36, a13 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v35, a14 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_read_b32 v34, a15 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_mov_b32 a32, a1
-; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v34, v33, a[16:31]
+; GFX90A-NEXT: v_accvgpr_mov_b32 a16, a1
+; GFX90A-NEXT: buffer_load_dword a0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX90A-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, v39 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, v38 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, v37 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, v36 ; Reload Reuse
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, v35 ; Reload Reuse
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, v34 ; Reload Reuse
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; copy
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_write_b32 a32, v35 ; Reload Reuse
-; GFX90A-NEXT: s_nop 9
; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a3 v[0:31]
@@ -1158,9 +1265,9 @@ define void @no_free_vgprs_at_sgpr_to_agpr_copy(float %v0, float %v1) #0 {
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32 immarg, i32 immarg, i32 immarg) #1
declare i32 @llvm.amdgcn.workitem.id.x() #2
-attributes #0 = { "amdgpu-waves-per-eu"="6,6" nounwind }
+attributes #0 = { "amdgpu-waves-per-eu"="6,6" }
attributes #1 = { convergent nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
-attributes #3 = { "amdgpu-waves-per-eu"="7,7" "amdgpu-agpr-alloc"="0" nounwind }
-attributes #4 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-flat-work-group-size"="1024,1024" nounwind }
-attributes #5 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-agpr-alloc"="0" nounwind }
+attributes #3 = { "amdgpu-waves-per-eu"="7,7" "amdgpu-agpr-alloc"="0" }
+attributes #4 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-flat-work-group-size"="1024,1024" }
+attributes #5 = { "amdgpu-waves-per-eu"="6,6" "amdgpu-agpr-alloc"="0" }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
index 20a51fd0cc575..418d35c3ea414 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.large.mir
@@ -6,195 +6,201 @@
define amdgpu_kernel void @largeInterleave() #0 { ret void }
; GCN-LABEL: largeInterleave:
; GCN: ; %bb.0:
- ; GCN-NEXT: ; implicit-def: $vgpr16
- ; GCN-NEXT: ; implicit-def: $vgpr25
+ ; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
- ; GCN-NEXT: v_readfirstlane_b32 s17, v16
+ ; GCN-NEXT: v_readfirstlane_b32 s18, v0
+ ; GCN-NEXT: ; implicit-def: $vgpr9
; GCN-NEXT: ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
- ; GCN-NEXT: ; implicit-def: $vgpr17
+ ; GCN-NEXT: ; implicit-def: $vgpr1
+ ; GCN-NEXT: ; implicit-def: $vgpr4
+ ; GCN-NEXT: ; implicit-def: $sgpr16
; GCN-NEXT: ; implicit-def: $sgpr15
+ ; GCN-NEXT: ; implicit-def: $vgpr5
; GCN-NEXT: ; implicit-def: $sgpr8_sgpr9_sgpr10_sgpr11
- ; GCN-NEXT: s_lshl_b32 s18, s17, 7
- ; GCN-NEXT: ; implicit-def: $vgpr18
- ; GCN-NEXT: v_add_lshl_u32 v230, v18, s18, 1
- ; GCN-NEXT: v_lshl_add_u32 v25, s17, 4, v25
- ; GCN-NEXT: v_mul_lo_u32 v25, v25, s6
- ; GCN-NEXT: v_add_lshl_u32 v226, v25, v17, 1
- ; GCN-NEXT: v_add_u32_e32 v17, s15, v226
- ; GCN-NEXT: buffer_load_dwordx4 v[64:67], v226, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: ; implicit-def: $vgpr2
+ ; GCN-NEXT: ; implicit-def: $vgpr238
+ ; GCN-NEXT: ; implicit-def: $vgpr239
+ ; GCN-NEXT: ; implicit-def: $vgpr128_vgpr129_vgpr130_vgpr131
+ ; GCN-NEXT: ; implicit-def: $vgpr132_vgpr133_vgpr134_vgpr135
+ ; GCN-NEXT: ; implicit-def: $vgpr154_vgpr155_vgpr156_vgpr157
+ ; GCN-NEXT: ; implicit-def: $vgpr158_vgpr159_vgpr160_vgpr161
+ ; GCN-NEXT: ; implicit-def: $vgpr3
+ ; GCN-NEXT: ; implicit-def: $vgpr10
+ ; GCN-NEXT: ; implicit-def: $vgpr11
+ ; GCN-NEXT: ; implicit-def: $vgpr12
+ ; GCN-NEXT: ; implicit-def: $vgpr13
+ ; GCN-NEXT: v_add_u32_e32 v232, v3, v10
+ ; GCN-NEXT: v_add_u32_e32 v233, v3, v11
+ ; GCN-NEXT: v_lshl_add_u32 v0, s18, 4, v9
+ ; GCN-NEXT: v_mul_lo_u32 v0, v0, s6
+ ; GCN-NEXT: v_add_lshl_u32 v230, v0, v1, 1
+ ; GCN-NEXT: v_add_u32_e32 v0, s18, v4
+ ; GCN-NEXT: v_and_b32_e32 v0, 0x1fffffff, v0
+ ; GCN-NEXT: v_mul_lo_u32 v0, v0, s16
+ ; GCN-NEXT: v_add_lshl_u32 v140, v5, v0, 1
+ ; GCN-NEXT: v_add_u32_e32 v0, s15, v230
+ ; GCN-NEXT: buffer_load_dwordx4 v[64:67], v230, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx4 v[68:71], v17, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[68:71], v0, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_add_u32_e32 v72, 64, v17
- ; GCN-NEXT: ; implicit-def: $vgpr213
- ; GCN-NEXT: ; implicit-def: $vgpr152_vgpr153_vgpr154_vgpr155
- ; GCN-NEXT: ; implicit-def: $vgpr246
- ; GCN-NEXT: v_add_u32_e32 v188, 0x80, v17
- ; GCN-NEXT: ; implicit-def: $vgpr156_vgpr157_vgpr158_vgpr159
- ; GCN-NEXT: ; implicit-def: $vgpr144_vgpr145_vgpr146_vgpr147
- ; GCN-NEXT: ; implicit-def: $vgpr19
- ; GCN-NEXT: ; implicit-def: $vgpr26
- ; GCN-NEXT: ; implicit-def: $vgpr27
- ; GCN-NEXT: v_add_u32_e32 v227, 0xc0, v17
- ; GCN-NEXT: v_add_u32_e32 v231, v19, v26
- ; GCN-NEXT: v_add_u32_e32 v232, v19, v27
+ ; GCN-NEXT: s_lshl_b32 s6, s18, 7
+ ; GCN-NEXT: v_add_lshl_u32 v240, v2, s6, 1
+ ; GCN-NEXT: v_add_u32_e32 v72, 64, v0
+ ; GCN-NEXT: v_add_u32_e32 v92, 0x80, v0
+ ; GCN-NEXT: v_add_u32_e32 v231, 0xc0, v0
+ ; GCN-NEXT: v_add_u32_e32 v234, v3, v12
+ ; GCN-NEXT: v_add_u32_e32 v235, v3, v13
; GCN-NEXT: ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3
- ; GCN-NEXT: ; implicit-def: $vgpr28
- ; GCN-NEXT: ; implicit-def: $vgpr29
- ; GCN-NEXT: v_add_u32_e32 v233, v19, v28
- ; GCN-NEXT: v_add_u32_e32 v234, v19, v29
- ; GCN-NEXT: ; implicit-def: $vgpr140_vgpr141_vgpr142_vgpr143
+ ; GCN-NEXT: ; implicit-def: $vgpr162_vgpr163_vgpr164_vgpr165
+ ; GCN-NEXT: ; implicit-def: $vgpr166_vgpr167_vgpr168_vgpr169
+ ; GCN-NEXT: ; implicit-def: $vgpr170_vgpr171_vgpr172_vgpr173
+ ; GCN-NEXT: ; implicit-def: $vgpr174_vgpr175_vgpr176_vgpr177
+ ; GCN-NEXT: ; implicit-def: $sgpr14
; GCN-NEXT: ; implicit-def: $sgpr5
; GCN-NEXT: ; implicit-def: $sgpr7
- ; GCN-NEXT: ; implicit-def: $vgpr148_vgpr149_vgpr150_vgpr151
- ; GCN-NEXT: ; implicit-def: $vgpr136_vgpr137_vgpr138_vgpr139
- ; GCN-NEXT: ; implicit-def: $vgpr132_vgpr133_vgpr134_vgpr135
+ ; GCN-NEXT: ; implicit-def: $vgpr6
+ ; GCN-NEXT: v_lshl_add_u32 v141, v6, 1, v140
+ ; GCN-NEXT: ; implicit-def: $vgpr7
+ ; GCN-NEXT: v_lshl_add_u32 v142, v7, 1, v141
+ ; GCN-NEXT: ; implicit-def: $vgpr8
+ ; GCN-NEXT: v_lshl_add_u32 v143, v8, 1, v142
; GCN-NEXT: ; implicit-def: $vgpr20
- ; GCN-NEXT: v_add_u32_e32 v18, s17, v20
- ; GCN-NEXT: v_and_b32_e32 v18, 0x1fffffff, v18
- ; GCN-NEXT: ; implicit-def: $sgpr16
- ; GCN-NEXT: v_mul_lo_u32 v18, v18, s16
- ; GCN-NEXT: ; implicit-def: $vgpr21
- ; GCN-NEXT: v_add_lshl_u32 v199, v21, v18, 1
- ; GCN-NEXT: ; implicit-def: $vgpr22
- ; GCN-NEXT: v_lshl_add_u32 v200, v22, 1, v199
- ; GCN-NEXT: ; implicit-def: $vgpr23
- ; GCN-NEXT: v_lshl_add_u32 v201, v23, 1, v200
- ; GCN-NEXT: ; implicit-def: $vgpr24
- ; GCN-NEXT: v_lshl_add_u32 v202, v24, 1, v201
+ ; GCN-NEXT: v_add_u32_e32 v241, v3, v20
+ ; GCN-NEXT: ; implicit-def: $vgpr14
+ ; GCN-NEXT: ; implicit-def: $vgpr15
; GCN-NEXT: ; implicit-def: $vgpr16
- ; GCN-NEXT: ; implicit-def: $vgpr18
- ; GCN-NEXT: ; implicit-def: $vgpr20
- ; GCN-NEXT: ; implicit-def: $vgpr24
- ; GCN-NEXT: v_add_u32_e32 v247, v19, v24
- ; GCN-NEXT: v_add_u32_e32 v248, v19, v16
- ; GCN-NEXT: v_add_u32_e32 v249, v19, v18
- ; GCN-NEXT: v_add_u32_e32 v250, v19, v20
- ; GCN-NEXT: ; implicit-def: $vgpr128_vgpr129_vgpr130_vgpr131
- ; GCN-NEXT: ; implicit-def: $sgpr14
- ; GCN-NEXT: ; implicit-def: $vgpr196
+ ; GCN-NEXT: v_add_u32_e32 v242, v3, v14
+ ; GCN-NEXT: v_add_u32_e32 v243, v3, v15
+ ; GCN-NEXT: v_add_u32_e32 v244, v3, v16
+ ; GCN-NEXT: ; implicit-def: $vgpr136
; GCN-NEXT: ; implicit-def: $sgpr12_sgpr13
- ; GCN-NEXT: ; implicit-def: $vgpr211
- ; GCN-NEXT: v_max_f32_e32 v212, v211, v211
- ; GCN-NEXT: ; implicit-def: $vgpr198
- ; GCN-NEXT: ; implicit-def: $vgpr32
- ; GCN-NEXT: ; implicit-def: $vgpr33
- ; GCN-NEXT: ; implicit-def: $vgpr34
- ; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
- ; GCN-NEXT: v_add_u32_e32 v210, v19, v34
- ; GCN-NEXT: v_add_u32_e32 v206, v19, v33
- ; GCN-NEXT: v_add_u32_e32 v205, v19, v32
- ; GCN-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+ ; GCN-NEXT: ; implicit-def: $vgpr153
+ ; GCN-NEXT: v_max_f32_e32 v148, v153, v153
+ ; GCN-NEXT: ; implicit-def: $vgpr139
+ ; GCN-NEXT: ; implicit-def: $vgpr17
+ ; GCN-NEXT: ; implicit-def: $vgpr18
+ ; GCN-NEXT: ; implicit-def: $vgpr19
; GCN-NEXT: ; implicit-def: $vgpr21
; GCN-NEXT: ; implicit-def: $vgpr22
; GCN-NEXT: ; implicit-def: $vgpr23
- ; GCN-NEXT: ; implicit-def: $vgpr30
- ; GCN-NEXT: ; implicit-def: $vgpr31
- ; GCN-NEXT: v_add_u32_e32 v207, v19, v21
- ; GCN-NEXT: v_add_u32_e32 v208, v19, v22
- ; GCN-NEXT: v_add_u32_e32 v209, v19, v23
- ; GCN-NEXT: v_add_u32_e32 v203, v19, v30
- ; GCN-NEXT: v_add_u32_e32 v204, v19, v31
- ; GCN-NEXT: ; kill: killed $vgpr17
- ; GCN-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; GCN-NEXT: ; implicit-def: $vgpr24
+ ; GCN-NEXT: ; implicit-def: $vgpr25
+ ; GCN-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+ ; GCN-NEXT: ; implicit-def: $sgpr17
+ ; GCN-NEXT: v_add_u32_e32 v20, s17, v20
+ ; GCN-NEXT: v_add_u32_e32 v16, s17, v16
+ ; GCN-NEXT: v_add_u32_e32 v152, v3, v25
+ ; GCN-NEXT: v_add_u32_e32 v25, s17, v25
+ ; GCN-NEXT: v_add_u32_e32 v149, v3, v17
+ ; GCN-NEXT: v_add_u32_e32 v17, s17, v17
+ ; GCN-NEXT: v_add_u32_e32 v150, v3, v18
+ ; GCN-NEXT: v_add_u32_e32 v18, s17, v18
+ ; GCN-NEXT: v_add_u32_e32 v151, v3, v19
+ ; GCN-NEXT: v_add_u32_e32 v19, s17, v19
+ ; GCN-NEXT: v_add_u32_e32 v147, v3, v24
+ ; GCN-NEXT: v_add_u32_e32 v24, s17, v24
+ ; GCN-NEXT: v_add_u32_e32 v145, v3, v21
+ ; GCN-NEXT: v_add_u32_e32 v21, s17, v21
+ ; GCN-NEXT: v_add_u32_e32 v144, v3, v22
+ ; GCN-NEXT: v_add_u32_e32 v22, s17, v22
+ ; GCN-NEXT: v_add_u32_e32 v146, v3, v23
+ ; GCN-NEXT: v_add_u32_e32 v23, s17, v23
; GCN-NEXT: ; implicit-def: $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
- ; GCN-NEXT: ; implicit-def: $vgpr197
+ ; GCN-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+ ; GCN-NEXT: ; implicit-def: $vgpr138
+ ; GCN-NEXT: ; implicit-def: $vgpr137
; GCN-NEXT: ; iglp_opt mask(0x00000002)
+ ; GCN-NEXT: ; kill: killed $vgpr145
+ ; GCN-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GCN-NEXT: ; kill: killed $vgpr146
+ ; GCN-NEXT: ; kill: killed $vgpr144
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[64:67]
+ ; GCN-NEXT: ds_write_b128 v240, v[64:67]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[68:71] offset:1024
+ ; GCN-NEXT: ds_write_b128 v240, v[68:71] offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx4 v[160:163], v226, s[8:11], 0 offen offset:64 sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[80:83], v230, s[8:11], 0 offen offset:64 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx4 v[164:167], v72, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[84:87], v72, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: ds_read_b128 v[64:67], v213
+ ; GCN-NEXT: ds_read_b128 v[64:67], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[64:65], v[152:153], 0
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[66:67], v[154:155], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[64:67], v213 offset:512
+ ; GCN-NEXT: ds_read_b128 v[68:71], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[64:65], v[152:153], 0
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[66:67], v[154:155], v[96:111]
- ; GCN-NEXT: ds_read_b128 v[64:67], v213 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[178:181], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[168:171], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[88:91], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[172:175], v246
+ ; GCN-NEXT: ds_read_b128 v[72:75], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[176:179], v246 offset:512
+ ; GCN-NEXT: ds_read_b128 v[76:79], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[180:183], v246 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[182:185], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[184:187], v246 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[64:65], v[152:153], 0
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[64:65], v[128:129], 0
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[160:163]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[66:67], v[154:155], v[80:95]
+ ; GCN-NEXT: ds_write_b128 v240, v[80:83]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[164:167] offset:1024
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[168:169], v[152:153], 0
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[170:171], v[154:155], v[64:79]
+ ; GCN-NEXT: ds_write_b128 v240, v[84:87] offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx4 v[152:155], v226, s[8:11], 0 offen offset:128 sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[190:193], v230, s[8:11], 0 offen offset:128 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx4 v[160:163], v188, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[194:197], v92, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[68:69], v[128:129], 0
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: ds_read_b128 v[188:191], v213
+ ; GCN-NEXT: ds_read_b128 v[198:201], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[192:195], v213 offset:512
+ ; GCN-NEXT: ds_read_b128 v[202:205], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[164:167], v213 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[206:209], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[214:217], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[210:213], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[172:173], v[156:157], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[218:221], v246
+ ; GCN-NEXT: ds_read_b128 v[214:217], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[222:225], v246 offset:512
+ ; GCN-NEXT: ds_read_b128 v[218:221], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[168:171], v246 offset:1024
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[66:67], v[130:131], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[222:225], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[174:175], v[158:159], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[188:189], v[144:145], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[190:191], v[146:147], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[188:191], v246 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[226:229], v239 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -202,81 +208,100 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[152:155]
+ ; GCN-NEXT: ds_write_b128 v240, v[190:193]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[70:71], v[130:131], v[96:111]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[160:163] offset:1024
+ ; GCN-NEXT: ds_write_b128 v240, v[194:197] offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx4 v[152:155], v226, s[8:11], 0 offen offset:192 sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[190:193], v230, s[8:11], 0 offen offset:192 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[184:185], v[156:157], v[64:79]
- ; GCN-NEXT: buffer_load_dwordx4 v[226:229], v227, s[8:11], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx4 v[194:197], v231, s[8:11], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[160:161], v231, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[72:73], v[132:133], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[76:77], v[132:133], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[74:75], v[134:135], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[78:79], v[134:135], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[88:89], v[128:129], 0
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[90:91], v[130:131], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[178:179], v[128:129], 0
+ ; GCN-NEXT: buffer_load_dwordx2 v[128:129], v232, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[162:163], v232, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[178:179], v233, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[172:173], v233, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[230:231], v234, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[174:175], v234, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v234, v178, v128, s5
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[198:199], v[154:155], v[112:127]
+ ; GCN-NEXT: buffer_load_dwordx2 v[232:233], v235, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[158:159], v[64:79]
- ; GCN-NEXT: v_perm_b32 v238, v162, v160, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[218:219], v[140:141], v[112:127]
- ; GCN-NEXT: v_perm_b32 v240, v162, v160, s7
- ; GCN-NEXT: v_perm_b32 v242, v163, v161, s5
- ; GCN-NEXT: v_perm_b32 v244, v163, v161, s7
- ; GCN-NEXT: ds_read_b128 v[160:163], v213
+ ; GCN-NEXT: v_perm_b32 v236, v178, v128, s7
+ ; GCN-NEXT: v_perm_b32 v128, v179, v129, s5
+ ; GCN-NEXT: v_perm_b32 v178, v179, v129, s7
+ ; GCN-NEXT: v_perm_b32 v235, v232, v230, s5
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[132:133], v[64:79]
+ ; GCN-NEXT: v_perm_b32 v237, v232, v230, s7
+ ; GCN-NEXT: v_perm_b32 v129, v233, v231, s5
+ ; GCN-NEXT: v_perm_b32 v179, v233, v231, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[180:181], v[130:131], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[200:201], v[156:157], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[198:201], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_perm_b32 v239, v174, v172, s5
- ; GCN-NEXT: v_perm_b32 v241, v174, v172, s7
- ; GCN-NEXT: v_perm_b32 v243, v175, v173, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[214:215], v[144:145], v[64:79]
- ; GCN-NEXT: v_perm_b32 v245, v175, v173, s7
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[176:177], v[156:157], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[220:221], v[142:143], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[218:221], v213 offset:512
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[134:135], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[182:183], v[132:133], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[214:215], v[158:159], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[210:211], v[154:155], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[202:203], v[154:155], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[184:185], v[134:135], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[216:217], v[160:161], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[214:217], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[172:175], v213 offset:1024
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[212:213], v[156:157], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[204:205], v[156:157], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[206:207], v[154:155], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[198:199], v[162:163], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[226:227], v[158:159], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[218:219], v[158:159], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[208:209], v[156:157], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[200:201], v[164:165], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[198:201], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[216:217], v[146:147], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[178:179], v[158:159], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[160:161], v[148:149], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[140:141], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[192:193], v[144:145], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[162:163], v[150:151], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[160:163], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[186:189], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[184:187], v246
+ ; GCN-NEXT: ds_read_b128 v[210:213], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[214:217], v246 offset:512
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[228:229], v[160:161], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[220:221], v[160:161], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[222:223], v[158:159], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[162:163], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[214:215], v[162:163], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[224:225], v[160:161], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[210:211], v[166:167], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[164:165], v[64:79]
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[176:179], v246 offset:1024
+ ; GCN-NEXT: ds_read_b128 v[226:229], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[190:191], v[142:143], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[194:195], v[146:147], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[160:161], v[148:149], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[180:181], v[156:157], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[184:185], v[136:137], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[222:223], v[140:141], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[162:163], v[150:151], v[64:79]
- ; GCN-NEXT: ds_read_b128 v[160:163], v246 offset:1536
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[216:217], v[164:165], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[198:199], v[162:163], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[212:213], v[168:169], v[112:127]
+ ; GCN-NEXT: ds_read_b128 v[210:213], v239 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -284,278 +309,259 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[152:155]
+ ; GCN-NEXT: ds_write_b128 v240, v[190:193]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b128 v230, v[226:229] offset:1024
+ ; GCN-NEXT: ds_write_b128 v240, v[194:197] offset:1024
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[182:183], v[158:159], v[80:95]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[156:159], v213
+ ; GCN-NEXT: ds_read_b128 v[190:193], v238
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[226:229], v213 offset:512
+ ; GCN-NEXT: ds_read_b128 v[194:197], v238 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[180:183], v213 offset:1024
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[186:187], v[166:167], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[202:205], v238 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[152:155], v213 offset:1536
+ ; GCN-NEXT: ds_read_b128 v[218:221], v238 offset:1536
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[230:233], v246
+ ; GCN-NEXT: ds_read_b128 v[214:217], v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[234:237], v246 offset:512
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[200:201], v[164:165], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[188:189], v[168:169], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[186:187], v[138:139], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[184:187], v246 offset:1024
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[224:225], v[142:143], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[156:157], v[132:133], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[218:219], v[148:149], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[158:159], v[134:135], v[112:127]
- ; GCN-NEXT: ds_read_b128 v[156:159], v246 offset:1536
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[190:191], v[170:171], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[226:227], v[166:167], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[194:195], v[170:171], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[192:193], v[172:173], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[228:229], v[168:169], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[210:211], v[166:167], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[196:197], v[172:173], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[194:197], v239 offset:1024
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[214:215], v[174:175], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[202:203], v[170:171], v[80:95]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[212:213], v[168:169], v[64:79]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[186:187], v[174:175], v[96:111]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[216:217], v[176:177], v[112:127]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[204:205], v[172:173], v[80:95]
+ ; GCN-NEXT: s_nop 9
+ ; GCN-NEXT: v_mul_f32_e32 v154, s4, v112
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v113
+ ; GCN-NEXT: v_max3_f32 v154, v154, s14, v155
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v114
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v115
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[218:219], v[170:171], v[64:79]
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v116
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v117
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v118
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v119
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[188:189], v[176:177], v[96:111]
+ ; GCN-NEXT: ds_read_b128 v[186:189], v239 offset:1536
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v120
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v121
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v122
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v123
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[194:195], v[174:175], v[80:95]
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v124
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v125
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v126
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v127
+ ; GCN-NEXT: v_mul_f32_e32 v190, s4, v96
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[220:221], v[172:173], v[64:79]
+ ; GCN-NEXT: v_mul_f32_e32 v191, s4, v97
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[238:239]
+ ; GCN-NEXT: ds_write_b64 v140, v[234:235]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[240:241]
+ ; GCN-NEXT: ds_write_b64 v141, v[236:237]
+ ; GCN-NEXT: v_mul_f32_e32 v236, s4, v98
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[196:197], v[176:177], v[80:95]
+ ; GCN-NEXT: v_mul_f32_e32 v237, s4, v99
+ ; GCN-NEXT: v_max3_f32 v154, v154, v190, v191
+ ; GCN-NEXT: v_mul_f32_e32 v238, s4, v100
+ ; GCN-NEXT: v_mul_f32_e32 v239, s4, v101
+ ; GCN-NEXT: v_max3_f32 v154, v154, v236, v237
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[242:243]
+ ; GCN-NEXT: ds_write_b64 v142, v[128:129]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[186:187], v[174:175], v[64:79]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[244:245]
+ ; GCN-NEXT: ds_write_b64 v143, v[178:179]
+ ; GCN-NEXT: v_mul_f32_e32 v192, s4, v102
+ ; GCN-NEXT: v_mul_f32_e32 v193, s4, v103
+ ; GCN-NEXT: v_max3_f32 v154, v154, v238, v239
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx2 v[192:193], v247, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[178:179], v241, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[220:221], v[150:151], v[96:111]
- ; GCN-NEXT: buffer_load_dwordx2 v[194:195], v248, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v240, s4, v104
+ ; GCN-NEXT: v_mul_f32_e32 v241, s4, v105
+ ; GCN-NEXT: v_max3_f32 v154, v154, v192, v193
+ ; GCN-NEXT: buffer_load_dwordx2 v[230:231], v242, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[218:219], v249, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[232:233], v243, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[220:221], v250, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[234:235], v244, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v242, s4, v106
+ ; GCN-NEXT: v_max3_f32 v154, v154, v240, v241
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v107
+ ; GCN-NEXT: v_max3_f32 v154, v154, v242, v155
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v108
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v109
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v110
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v111
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[188:189], v[176:177], v[64:79]
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v80
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v81
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v82
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v83
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v84
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v85
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v86
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v87
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v88
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v89
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v90
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v91
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v92
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v93
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v94
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v95
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v64
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v65
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v66
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v67
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v68
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v69
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v70
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v71
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v72
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v73
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v74
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v75
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v76
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v77
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: v_mul_f32_e32 v155, s4, v78
+ ; GCN-NEXT: v_mul_f32_e32 v156, s4, v79
+ ; GCN-NEXT: v_max3_f32 v154, v154, v155, v156
+ ; GCN-NEXT: ds_bpermute_b32 v155, v136, v154
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_perm_b32 v188, v194, v192, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[164:165], v[144:145], v[80:95]
- ; GCN-NEXT: v_perm_b32 v189, v220, v218, s5
- ; GCN-NEXT: v_perm_b32 v191, v220, v218, s7
- ; GCN-NEXT: v_perm_b32 v190, v194, v192, s7
- ; GCN-NEXT: v_perm_b32 v192, v195, v193, s5
- ; GCN-NEXT: v_perm_b32 v194, v195, v193, s7
- ; GCN-NEXT: v_perm_b32 v193, v221, v219, s5
- ; GCN-NEXT: v_perm_b32 v195, v221, v219, s7
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[166:167], v[146:147], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[168:169], v[140:141], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[170:171], v[142:143], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[172:173], v[148:149], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[214:215], v[136:137], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[174:175], v[150:151], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[216:217], v[138:139], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[176:177], v[136:137], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[226:227], v[132:133], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[178:179], v[138:139], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[160:161], v[136:137], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[230:231], v[128:129], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[228:229], v[134:135], v[96:111]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[180:181], v[132:133], v[80:95]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[162:163], v[138:139], v[64:79]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[112:127], v[232:233], v[130:131], v[112:127]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[234:235], v[128:129], v[96:111]
- ; GCN-NEXT: s_nop 9
- ; GCN-NEXT: v_mul_f32_e32 v213, s4, v112
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v113
- ; GCN-NEXT: v_max3_f32 v213, v213, s14, v218
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v114
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v115
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v116
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[182:183], v[134:135], v[80:95]
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v117
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v118
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v119
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v120
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v121
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[152:153], v[132:133], v[64:79]
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v122
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v123
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v124
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v125
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[96:111], v[236:237], v[130:131], v[96:111]
- ; GCN-NEXT: v_mul_f32_e32 v218, s4, v126
- ; GCN-NEXT: v_mul_f32_e32 v219, s4, v127
- ; GCN-NEXT: v_max3_f32 v213, v213, v218, v219
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[184:185], v[128:129], v[80:95]
- ; GCN-NEXT: s_nop 6
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v96
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v97
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v98
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v99
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v100
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[154:155], v[134:135], v[64:79]
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v101
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v102
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v103
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v104
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v105
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[80:95], v[186:187], v[130:131], v[80:95]
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v106
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v107
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v108
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v109
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[156:157], v[128:129], v[64:79]
- ; GCN-NEXT: v_mul_f32_e32 v214, s4, v110
- ; GCN-NEXT: v_mul_f32_e32 v215, s4, v111
- ; GCN-NEXT: v_max3_f32 v213, v213, v214, v215
- ; GCN-NEXT: v_mul_f32_e32 v140, s4, v80
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v81
- ; GCN-NEXT: v_max3_f32 v140, v213, v140, v141
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v82
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[64:79], v[158:159], v[130:131], v[64:79]
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v83
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v84
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v85
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v86
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v87
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v88
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v89
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v90
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v91
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v92
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v93
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v141, s4, v94
- ; GCN-NEXT: v_mul_f32_e32 v142, s4, v95
- ; GCN-NEXT: v_max3_f32 v140, v140, v141, v142
- ; GCN-NEXT: v_mul_f32_e32 v128, s4, v64
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v65
- ; GCN-NEXT: v_max3_f32 v128, v140, v128, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v66
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v67
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v68
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v69
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v70
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v71
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v72
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v73
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v74
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v75
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v76
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v77
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: v_mul_f32_e32 v129, s4, v78
- ; GCN-NEXT: v_mul_f32_e32 v130, s4, v79
- ; GCN-NEXT: v_max3_f32 v128, v128, v129, v130
- ; GCN-NEXT: ds_bpermute_b32 v129, v196, v128
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[130:133], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_max_f32_e32 v129, v129, v129
- ; GCN-NEXT: v_max_f32_e32 v128, v128, v129
- ; GCN-NEXT: ds_bpermute_b32 v129, v196, v128
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: v_cndmask_b32_e64 v128, v129, v128, s[12:13]
- ; GCN-NEXT: v_max_f32_e32 v128, v128, v128
- ; GCN-NEXT: v_max_f32_e32 v128, v212, v128
- ; GCN-NEXT: v_fma_f32 v113, s4, v113, -v128
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v114, -v128
- ; GCN-NEXT: v_mul_f32_e32 v139, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v115, -v128
- ; GCN-NEXT: v_mul_f32_e32 v140, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v116, -v128
- ; GCN-NEXT: v_mul_f32_e32 v141, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v117, -v128
- ; GCN-NEXT: v_fma_f32 v112, s4, v112, -v128
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v118, -v128
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_max_f32_e32 v155, v155, v155
+ ; GCN-NEXT: v_max_f32_e32 v154, v154, v155
+ ; GCN-NEXT: ds_bpermute_b32 v155, v136, v154
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_cndmask_b32_e64 v154, v155, v154, s[12:13]
+ ; GCN-NEXT: v_max_f32_e32 v154, v154, v154
+ ; GCN-NEXT: v_max_f32_e32 v148, v148, v154
+ ; GCN-NEXT: v_fma_f32 v112, s4, v112, -v148
; GCN-NEXT: v_mul_f32_e32 v112, 0x3fb8aa3b, v112
- ; GCN-NEXT: v_mul_f32_e32 v143, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_fma_f32 v113, s4, v119, -v128
- ; GCN-NEXT: v_mul_f32_e32 v144, 0x3fb8aa3b, v113
- ; GCN-NEXT: v_exp_f32_e32 v113, v112
- ; GCN-NEXT: v_exp_f32_e32 v114, v138
- ; GCN-NEXT: v_exp_f32_e32 v115, v139
- ; GCN-NEXT: v_exp_f32_e32 v116, v140
- ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v113
- ; GCN-NEXT: v_cvt_f16_f32_e32 v119, v114
- ; GCN-NEXT: v_fma_f32 v118, s4, v120, -v128
- ; GCN-NEXT: v_fma_f32 v120, s4, v121, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v121, v116
- ; GCN-NEXT: v_pack_b32_f16 v146, v112, v119
- ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v115
- ; GCN-NEXT: v_sub_f32_e32 v129, v211, v128
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v129
- ; GCN-NEXT: v_mul_f32_e32 v149, 0x3fb8aa3b, v120
- ; GCN-NEXT: v_fma_f32 v120, s4, v122, -v128
- ; GCN-NEXT: v_exp_f32_e32 v117, v141
- ; GCN-NEXT: v_mul_f32_e32 v148, 0x3fb8aa3b, v118
- ; GCN-NEXT: v_exp_f32_e32 v118, v142
- ; GCN-NEXT: v_exp_f32_e32 v119, v143
- ; GCN-NEXT: v_mul_f32_e32 v150, 0x3fb8aa3b, v120
- ; GCN-NEXT: v_exp_f32_e32 v120, v144
- ; GCN-NEXT: v_pack_b32_f16 v147, v112, v121
- ; GCN-NEXT: v_exp_f32_e32 v112, v129
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_fma_f32 v122, s4, v123, -v128
- ; GCN-NEXT: v_mul_f32_e32 v151, 0x3fb8aa3b, v122
- ; GCN-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[10:11], v[10:11], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[12:13], v[12:13], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[14:15], v[14:15], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_fma_f32 v113, s4, v113, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v113, 0x3fb8aa3b, v113
+ ; GCN-NEXT: v_fma_f32 v114, s4, v114, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v112, v112
+ ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v115, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v113, v113
+ ; GCN-NEXT: v_mul_f32_e32 v159, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v116, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v158, v158
+ ; GCN-NEXT: v_mul_f32_e32 v160, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v117, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v159, v159
+ ; GCN-NEXT: v_mul_f32_e32 v161, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_add_f32_e32 v162, 0, v112
+ ; GCN-NEXT: v_exp_f32_e32 v164, v160
+ ; GCN-NEXT: v_add_f32_e32 v162, v113, v162
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v112
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v113, v113
+ ; GCN-NEXT: v_exp_f32_e32 v166, v161
+ ; GCN-NEXT: v_add_f32_e32 v162, v158, v162
+ ; GCN-NEXT: v_fma_f32 v114, s4, v118, -v148
+ ; GCN-NEXT: v_add_f32_e32 v162, v159, v162
+ ; GCN-NEXT: v_fma_f32 v120, s4, v120, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v118, 0x3fb8aa3b, v114
+ ; GCN-NEXT: v_fma_f32 v114, s4, v119, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v165, 0x3fb8aa3b, v120
+ ; GCN-NEXT: v_fma_f32 v120, s4, v121, -v148
+ ; GCN-NEXT: v_add_f32_e32 v121, v164, v162
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v114
+ ; GCN-NEXT: ds_read_b128 v[114:117], v139
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v162, v112, v113
+ ; GCN-NEXT: v_mul_f32_e32 v113, 0x3fb8aa3b, v120
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v112, v158
+ ; GCN-NEXT: v_add_f32_e32 v120, v166, v121
+ ; GCN-NEXT: v_fma_f32 v121, s4, v122, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v122, v118
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v118, v159
+ ; GCN-NEXT: ds_read_b128 v[154:157], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mul_f32_e32 v167, 0x3fb8aa3b, v121
+ ; GCN-NEXT: v_fma_f32 v121, s4, v123, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v123, v119
+ ; GCN-NEXT: v_sub_f32_e32 v153, v153, v148
+ ; GCN-NEXT: v_mul_f32_e32 v153, 0x3fb8aa3b, v153
+ ; GCN-NEXT: v_add_f32_e32 v120, v122, v120
+ ; GCN-NEXT: v_pack_b32_f16 v163, v112, v118
+ ; GCN-NEXT: v_exp_f32_e32 v112, v153
+ ; GCN-NEXT: v_mul_f32_e32 v168, 0x3fb8aa3b, v121
+ ; GCN-NEXT: v_add_f32_e32 v153, v123, v120
+ ; GCN-NEXT: ds_read_b128 v[118:121], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[158:161], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: v_pk_mul_f32 v[32:33], v[32:33], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[34:35], v[34:35], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[36:37], v[36:37], v[112:113] op_sel_hi:[1,0]
@@ -564,234 +570,268 @@
; GCN-NEXT: v_pk_mul_f32 v[42:43], v[42:43], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[44:45], v[44:45], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[46:47], v[46:47], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v123, v117
- ; GCN-NEXT: v_fma_f32 v122, s4, v124, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v124, v118
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[146:147], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v121, v148
- ; GCN-NEXT: v_pk_mul_f32 v[16:17], v[16:17], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[18:19], v[18:19], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[20:21], v[20:21], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[22:23], v[22:23], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[24:25], v[24:25], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pk_mul_f32 v[26:27], v[26:27], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[134:135], v[146:147], v[32:47]
- ; GCN-NEXT: v_mul_f32_e64 v28, v28, v112
- ; GCN-NEXT: v_mul_f32_e64 v29, v29, v112
- ; GCN-NEXT: v_mul_f32_e64 v30, v30, v112
- ; GCN-NEXT: v_mul_f32_e64 v31, v31, v112
- ; GCN-NEXT: v_mul_f32_e64 v48, v48, v112
- ; GCN-NEXT: v_mul_f32_e64 v49, v49, v112
+ ; GCN-NEXT: v_pk_mul_f32 v[48:49], v[48:49], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[50:51], v[50:51], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[114:115], v[162:163], v[32:47]
; GCN-NEXT: v_pk_mul_f32 v[52:53], v[52:53], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[54:55], v[54:55], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[56:57], v[56:57], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[58:59], v[58:59], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[60:61], v[60:61], v[112:113] op_sel_hi:[1,0]
; GCN-NEXT: v_pk_mul_f32 v[62:63], v[62:63], v[112:113] op_sel_hi:[1,0]
- ; GCN-NEXT: v_pack_b32_f16 v134, v123, v124
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v119
- ; GCN-NEXT: v_fma_f32 v124, s4, v126, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v120
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v122
- ; GCN-NEXT: v_exp_f32_e32 v122, v149
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[138:139], v[146:147], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v123, v150
- ; GCN-NEXT: v_pack_b32_f16 v135, v130, v126
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v124
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v121
- ; GCN-NEXT: v_fma_f32 v139, s4, v96, -v128
- ; GCN-NEXT: v_fma_f32 v125, s4, v125, -v128
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v125
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[142:143], v[146:147], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v124, v151
- ; GCN-NEXT: v_fma_f32 v127, s4, v127, -v128
- ; GCN-NEXT: v_mul_f32_e32 v127, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_fma_f32 v143, s4, v101, -v128
- ; GCN-NEXT: v_fma_f32 v64, s4, v64, -v128
- ; GCN-NEXT: v_fma_f32 v65, s4, v65, -v128
- ; GCN-NEXT: v_fma_f32 v68, s4, v68, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[134:135], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v96, v129
- ; GCN-NEXT: ds_read_b128 v[130:133], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[146:149], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_fma_f32 v69, s4, v69, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[136:137], v[134:135], v[32:47]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v122
- ; GCN-NEXT: v_fma_f32 v137, s4, v97, -v128
- ; GCN-NEXT: v_exp_f32_e32 v97, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_pack_b32_f16 v126, v126, v136
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v123
- ; GCN-NEXT: v_fma_f32 v137, s4, v98, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[140:141], v[134:135], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v98, v138
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[144:145], v[134:135], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v124
- ; GCN-NEXT: v_fma_f32 v135, s4, v99, -v128
- ; GCN-NEXT: v_exp_f32_e32 v99, v127
- ; GCN-NEXT: v_mul_f32_e32 v150, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_pack_b32_f16 v127, v136, v134
- ; GCN-NEXT: ds_read_b128 v[134:137], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[138:141], v197 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[126:127], v[0:15]
- ; GCN-NEXT: v_fma_f32 v131, s4, v100, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v96
- ; GCN-NEXT: v_exp_f32_e32 v100, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_cvt_f16_f32_e32 v131, v97
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v114, v164
+ ; GCN-NEXT: v_exp_f32_e32 v164, v165
+ ; GCN-NEXT: v_pk_mul_f32 v[16:17], v[16:17], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[18:19], v[18:19], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[20:21], v[20:21], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[22:23], v[22:23], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[24:25], v[24:25], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[26:27], v[26:27], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[28:29], v[28:29], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[30:31], v[30:31], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[10:11], v[10:11], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[12:13], v[12:13], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_pk_mul_f32 v[14:15], v[14:15], v[112:113] op_sel_hi:[1,0]
+ ; GCN-NEXT: v_fma_f32 v115, s4, v124, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[154:155], v[162:163], v[48:63]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v124, v166
+ ; GCN-NEXT: v_exp_f32_e32 v113, v113
+ ; GCN-NEXT: v_mul_f32_e32 v165, 0x3fb8aa3b, v115
+ ; GCN-NEXT: v_add_f32_e32 v115, v164, v153
+ ; GCN-NEXT: v_pack_b32_f16 v154, v114, v124
+ ; GCN-NEXT: v_add_f32_e32 v114, v113, v115
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[118:119], v[162:163], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v115, v122
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v122, v123
+ ; GCN-NEXT: v_exp_f32_e32 v119, v167
+ ; GCN-NEXT: v_fma_f32 v123, s4, v127, -v148
+ ; GCN-NEXT: v_fma_f32 v125, s4, v125, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v155, v115, v122
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[158:159], v[162:163], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v127, v168
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v159, v164
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v113, v113
+ ; GCN-NEXT: v_mul_f32_e32 v153, 0x3fb8aa3b, v125
+ ; GCN-NEXT: v_fma_f32 v118, s4, v126, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v118, 0x3fb8aa3b, v118
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[116:117], v[154:155], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v162, v165
+ ; GCN-NEXT: v_add_f32_e32 v114, v119, v114
+ ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v123
+ ; GCN-NEXT: v_add_f32_e32 v126, v127, v114
+ ; GCN-NEXT: v_fma_f32 v96, s4, v96, -v148
+ ; GCN-NEXT: v_fma_f32 v97, s4, v97, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[156:157], v[154:155], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v153, v153
+ ; GCN-NEXT: v_fma_f32 v98, s4, v98, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v163, 0x3fb8aa3b, v96
+ ; GCN-NEXT: v_add_f32_e32 v96, v162, v126
+ ; GCN-NEXT: v_pack_b32_f16 v126, v159, v113
+ ; GCN-NEXT: v_mul_f32_e32 v113, 0x3fb8aa3b, v97
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[120:121], v[154:155], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v156, v118
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v119
+ ; GCN-NEXT: v_mul_f32_e32 v157, 0x3fb8aa3b, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v98, v127
+ ; GCN-NEXT: v_add_f32_e32 v96, v153, v96
+ ; GCN-NEXT: v_add_f32_e32 v96, v156, v96
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[160:161], v[154:155], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v154, v158
+ ; GCN-NEXT: v_fma_f32 v99, s4, v99, -v148
+ ; GCN-NEXT: ds_read_b128 v[114:117], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[122:125], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v127, v97, v98
+ ; GCN-NEXT: v_mul_f32_e32 v155, 0x3fb8aa3b, v99
+ ; GCN-NEXT: v_add_f32_e32 v158, v154, v96
+ ; GCN-NEXT: ds_read_b128 v[96:99], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[118:121], v138 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v128, v230, v178, s5
+ ; GCN-NEXT: v_perm_b32 v129, v234, v232, s5
+ ; GCN-NEXT: v_perm_b32 v130, v230, v178, s7
+ ; GCN-NEXT: v_perm_b32 v131, v234, v232, s7
+ ; GCN-NEXT: v_perm_b32 v132, v231, v179, s5
+ ; GCN-NEXT: v_perm_b32 v134, v231, v179, s7
+ ; GCN-NEXT: v_perm_b32 v133, v235, v233, s5
+ ; GCN-NEXT: v_perm_b32 v135, v235, v233, s7
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[188:189]
+ ; GCN-NEXT: ds_write_b64 v140, v[128:129]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[190:191]
+ ; GCN-NEXT: ds_write_b64 v141, v[130:131]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[192:193]
+ ; GCN-NEXT: ds_write_b64 v142, v[132:133]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[194:195]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[146:147], v[126:127], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v101, v125
- ; GCN-NEXT: v_pack_b32_f16 v146, v130, v131
+ ; GCN-NEXT: ds_write_b64 v143, v[134:135]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[114:115], v[126:127], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v132, v163
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx2 v[130:131], v210, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[128:129], v152, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v143
- ; GCN-NEXT: v_cvt_f16_f32_e32 v147, v98
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[126:127], v[16:31]
- ; GCN-NEXT: v_fma_f32 v134, s4, v102, -v128
- ; GCN-NEXT: v_mul_f32_e32 v156, 0x3fb8aa3b, v134
- ; GCN-NEXT: buffer_load_dwordx2 v[134:135], v207, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v100, s4, v100, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v133, 0x3fb8aa3b, v100
+ ; GCN-NEXT: v_add_f32_e32 v100, v132, v158
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v114, v162
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[122:123], v[126:127], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v113, v113
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v115, v153
+ ; GCN-NEXT: v_fma_f32 v104, s4, v104, -v148
+ ; GCN-NEXT: v_fma_f32 v101, s4, v101, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v134, 0x3fb8aa3b, v101
+ ; GCN-NEXT: v_pack_b32_f16 v122, v114, v115
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[96:97], v[126:127], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v135, v157
+ ; GCN-NEXT: v_add_f32_e32 v96, v113, v100
+ ; GCN-NEXT: v_fma_f32 v100, s4, v102, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v156
+ ; GCN-NEXT: v_mul_f32_e32 v153, 0x3fb8aa3b, v100
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v100, v154
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[118:119], v[126:127], v[0:15]
+ ; GCN-NEXT: buffer_load_dwordx2 v[118:119], v149, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_exp_f32_e32 v102, v142
- ; GCN-NEXT: buffer_load_dwordx2 v[142:143], v208, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[126:127], v150, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[144:145], v209, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[130:131], v151, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v123, v97, v100
+ ; GCN-NEXT: v_exp_f32_e32 v152, v155
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v132
+ ; GCN-NEXT: v_fma_f32 v101, s4, v103, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[116:117], v[122:123], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v132, v133
+ ; GCN-NEXT: v_mul_f32_e32 v133, 0x3fb8aa3b, v104
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v104, v113
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[126:127], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v99
- ; GCN-NEXT: v_fma_f32 v127, s4, v103, -v128
- ; GCN-NEXT: v_exp_f32_e32 v103, v150
- ; GCN-NEXT: v_fma_f32 v139, s4, v105, -v128
- ; GCN-NEXT: v_pack_b32_f16 v147, v147, v126
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_perm_b32 v152, v135, v131, s5
- ; GCN-NEXT: v_perm_b32 v154, v135, v131, s7
- ; GCN-NEXT: v_fma_f32 v135, s4, v104, -v128
- ; GCN-NEXT: v_perm_b32 v126, v134, v130, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[146:147], v[0:15]
- ; GCN-NEXT: v_perm_b32 v150, v134, v130, s7
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v100
- ; GCN-NEXT: v_exp_f32_e32 v104, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v101
- ; GCN-NEXT: ds_read_b128 v[130:133], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_perm_b32 v127, v144, v142, s5
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[148:149], v[146:147], v[32:47]
- ; GCN-NEXT: v_pack_b32_f16 v148, v134, v135
- ; GCN-NEXT: v_fma_f32 v135, s4, v106, -v128
- ; GCN-NEXT: v_exp_f32_e32 v105, v125
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v102
- ; GCN-NEXT: v_perm_b32 v151, v144, v142, s7
- ; GCN-NEXT: v_perm_b32 v153, v145, v143, s5
- ; GCN-NEXT: v_perm_b32 v155, v145, v143, s7
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[146:147], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v106, v156
- ; GCN-NEXT: v_mul_f32_e32 v156, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v103
- ; GCN-NEXT: v_fma_f32 v136, s4, v107, -v128
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_pack_b32_f16 v149, v134, v135
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[146:147], v[48:63]
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v136
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_exp_f32_e32 v107, v138
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[148:149], v[0:15]
- ; GCN-NEXT: v_fma_f32 v131, s4, v108, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v104
- ; GCN-NEXT: v_exp_f32_e32 v108, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_cvt_f16_f32_e32 v131, v105
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[142:143], v[148:149], v[32:47]
- ; GCN-NEXT: v_fma_f32 v142, s4, v109, -v128
- ; GCN-NEXT: v_exp_f32_e32 v109, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v142
- ; GCN-NEXT: v_pack_b32_f16 v142, v130, v131
- ; GCN-NEXT: v_fma_f32 v131, s4, v110, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v106
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[148:149], v[16:31]
- ; GCN-NEXT: v_mul_f32_e32 v134, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_cvt_f16_f32_e32 v131, v107
- ; GCN-NEXT: v_exp_f32_e32 v110, v156
- ; GCN-NEXT: v_fma_f32 v135, s4, v111, -v128
- ; GCN-NEXT: v_mul_f32_e32 v135, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_pack_b32_f16 v143, v130, v131
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[148:149], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v111, v146
- ; GCN-NEXT: v_fma_f32 v139, s4, v80, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v138, v108
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[142:143], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v80, v129
- ; GCN-NEXT: ds_read_b128 v[130:133], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[146:149], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_cvt_f16_f32_e32 v139, v109
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[144:145], v[142:143], v[32:47]
- ; GCN-NEXT: v_fma_f32 v144, s4, v81, -v128
- ; GCN-NEXT: v_exp_f32_e32 v81, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v144
- ; GCN-NEXT: v_pack_b32_f16 v144, v138, v139
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[142:143], v[16:31]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v110
- ; GCN-NEXT: v_fma_f32 v137, s4, v82, -v128
- ; GCN-NEXT: v_exp_f32_e32 v82, v134
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v111
- ; GCN-NEXT: v_mul_f32_e32 v156, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_fma_f32 v137, s4, v83, -v128
- ; GCN-NEXT: v_mul_f32_e32 v157, 0x3fb8aa3b, v137
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[142:143], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v83, v135
- ; GCN-NEXT: v_pack_b32_f16 v145, v136, v134
- ; GCN-NEXT: ds_read_b128 v[134:137], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[138:141], v197 offset:1728
+ ; GCN-NEXT: v_add_f32_e32 v96, v135, v96
+ ; GCN-NEXT: v_mul_f32_e32 v154, 0x3fb8aa3b, v101
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[124:125], v[122:123], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v113, v134
+ ; GCN-NEXT: ds_read_b128 v[100:103], v139
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v124, v97, v104
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v97, v135
+ ; GCN-NEXT: v_add_f32_e32 v96, v152, v96
+ ; GCN-NEXT: ds_read_b128 v[114:117], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[98:99], v[122:123], v[16:31]
+ ; GCN-NEXT: v_fma_f32 v98, s4, v106, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v135, v153
+ ; GCN-NEXT: v_mul_f32_e32 v149, 0x3fb8aa3b, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v98, v152
+ ; GCN-NEXT: v_add_f32_e32 v96, v132, v96
+ ; GCN-NEXT: v_add_f32_e32 v96, v113, v96
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[120:121], v[122:123], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v120, v154
+ ; GCN-NEXT: v_add_f32_e32 v96, v135, v96
+ ; GCN-NEXT: v_fma_f32 v99, s4, v107, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v125, v97, v98
+ ; GCN-NEXT: v_mul_f32_e32 v121, 0x3fb8aa3b, v99
+ ; GCN-NEXT: v_add_f32_e32 v122, v120, v96
+ ; GCN-NEXT: ds_read_b128 v[96:99], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v105, s4, v105, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v134, 0x3fb8aa3b, v105
+ ; GCN-NEXT: ds_read_b128 v[104:107], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[100:101], v[124:125], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v123, v133
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v100, v132
+ ; GCN-NEXT: v_fma_f32 v101, s4, v108, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v108, v113
+ ; GCN-NEXT: v_mul_f32_e32 v132, 0x3fb8aa3b, v101
+ ; GCN-NEXT: v_add_f32_e32 v101, v123, v122
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[114:115], v[124:125], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v113, v134
+ ; GCN-NEXT: v_pack_b32_f16 v114, v100, v108
+ ; GCN-NEXT: v_fma_f32 v109, s4, v109, -v148
+ ; GCN-NEXT: v_add_f32_e32 v100, v113, v101
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v101, v120
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v113, v113
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[96:97], v[124:125], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v96, v135
+ ; GCN-NEXT: v_exp_f32_e32 v134, v149
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v123
+ ; GCN-NEXT: v_pack_b32_f16 v115, v96, v101
+ ; GCN-NEXT: v_mul_f32_e32 v133, 0x3fb8aa3b, v109
+ ; GCN-NEXT: v_add_f32_e32 v100, v134, v100
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[104:105], v[124:125], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v124, v121
+ ; GCN-NEXT: v_fma_f32 v97, s4, v110, -v148
+ ; GCN-NEXT: v_fma_f32 v80, s4, v80, -v148
+ ; GCN-NEXT: v_add_f32_e32 v125, v124, v100
+ ; GCN-NEXT: v_mul_f32_e32 v97, 0x3fb8aa3b, v97
+ ; GCN-NEXT: v_fma_f32 v104, s4, v111, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[102:103], v[114:115], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v132, v132
+ ; GCN-NEXT: ds_read_b128 v[100:103], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v81, s4, v81, -v148
+ ; GCN-NEXT: v_fma_f32 v82, s4, v82, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v96, 0x3fb8aa3b, v104
+ ; GCN-NEXT: v_perm_b32 v105, v130, v126, s5
+ ; GCN-NEXT: v_perm_b32 v121, v130, v126, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[116:117], v[114:115], v[48:63]
+ ; GCN-NEXT: v_mul_f32_e32 v126, 0x3fb8aa3b, v80
+ ; GCN-NEXT: v_add_f32_e32 v80, v132, v125
+ ; GCN-NEXT: v_exp_f32_e32 v125, v133
+ ; GCN-NEXT: ds_read_b128 v[108:111], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v104, v118, v128, s5
+ ; GCN-NEXT: v_perm_b32 v120, v118, v128, s7
+ ; GCN-NEXT: v_perm_b32 v118, v119, v129, s5
+ ; GCN-NEXT: v_perm_b32 v122, v119, v129, s7
+ ; GCN-NEXT: v_perm_b32 v119, v131, v127, s5
+ ; GCN-NEXT: v_perm_b32 v123, v131, v127, s7
+ ; GCN-NEXT: v_mul_f32_e32 v127, 0x3fb8aa3b, v81
+ ; GCN-NEXT: v_pack_b32_f16 v116, v135, v113
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[98:99], v[114:115], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v134
+ ; GCN-NEXT: v_exp_f32_e32 v113, v97
+ ; GCN-NEXT: v_mul_f32_e32 v128, 0x3fb8aa3b, v82
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v82, v124
+ ; GCN-NEXT: v_add_f32_e32 v80, v125, v80
+ ; GCN-NEXT: v_add_f32_e32 v80, v113, v80
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[106:107], v[114:115], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v114, v96
+ ; GCN-NEXT: v_fma_f32 v83, s4, v83, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v117, v81, v82
+ ; GCN-NEXT: v_mul_f32_e32 v115, 0x3fb8aa3b, v83
+ ; GCN-NEXT: v_add_f32_e32 v106, v114, v80
+ ; GCN-NEXT: ds_read_b128 v[80:83], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[100:101], v[116:117], v[32:47]
+ ; GCN-NEXT: ds_read_b128 v[96:99], v138 offset:1728
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -799,150 +839,164 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[126:127]
+ ; GCN-NEXT: ds_write_b64 v140, v[104:105]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[150:151]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[144:145], v[0:15]
+ ; GCN-NEXT: ds_write_b64 v141, v[120:121]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[152:153]
+ ; GCN-NEXT: ds_write_b64 v142, v[118:119]
+ ; GCN-NEXT: v_exp_f32_e32 v118, v126
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[154:155]
- ; GCN-NEXT: v_fma_f32 v127, s4, v84, -v128
- ; GCN-NEXT: v_exp_f32_e32 v84, v129
- ; GCN-NEXT: v_fma_f32 v130, s4, v85, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v80
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[146:147], v[144:145], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v85, v125
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v130
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_load_dwordx2 v[130:131], v206, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: ds_write_b64 v143, v[122:123]
+ ; GCN-NEXT: v_fma_f32 v84, s4, v84, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[108:109], v[116:117], v[48:63]
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v84
+ ; GCN-NEXT: v_add_f32_e32 v84, v118, v106
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_load_dwordx2 v[106:107], v147, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v127, v81
- ; GCN-NEXT: v_pack_b32_f16 v126, v126, v127
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[144:145], v[16:31]
- ; GCN-NEXT: v_fma_f32 v134, s4, v86, -v128
- ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v134
- ; GCN-NEXT: buffer_load_dwordx2 v[134:135], v203, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[108:109], v145, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[142:143], v204, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: v_exp_f32_e32 v120, v127
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v100, v132
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[80:81], v[116:117], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v113
+ ; GCN-NEXT: v_add_f32_e32 v80, v120, v84
+ ; GCN-NEXT: v_fma_f32 v84, s4, v86, -v148
+ ; GCN-NEXT: v_exp_f32_e32 v113, v128
+ ; GCN-NEXT: v_mul_f32_e32 v122, 0x3fb8aa3b, v84
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v114
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[96:97], v[116:117], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v96, v115
+ ; GCN-NEXT: buffer_load_dwordx2 v[114:115], v144, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: buffer_load_dwordx2 v[146:147], v205, s[0:3], 0 offen sc0 sc1
+ ; GCN-NEXT: buffer_load_dwordx2 v[116:117], v146, s[0:3], 0 offen sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v127, v82
- ; GCN-NEXT: v_exp_f32_e32 v86, v156
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[144:145], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v138, v83
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v101, v125
+ ; GCN-NEXT: v_pack_b32_f16 v105, v81, v84
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v118
+ ; GCN-NEXT: v_fma_f32 v85, s4, v85, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v104, v100, v101
+ ; GCN-NEXT: v_add_f32_e32 v80, v113, v80
+ ; GCN-NEXT: v_mul_f32_e32 v121, 0x3fb8aa3b, v85
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[102:103], v[104:105], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v118, v119
+ ; GCN-NEXT: v_add_f32_e32 v80, v96, v80
+ ; GCN-NEXT: v_fma_f32 v88, s4, v88, -v148
+ ; GCN-NEXT: v_fma_f32 v85, s4, v87, -v148
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_fma_f32 v139, s4, v87, -v128
- ; GCN-NEXT: v_exp_f32_e32 v87, v157
- ; GCN-NEXT: v_pack_b32_f16 v127, v127, v138
- ; GCN-NEXT: v_fma_f32 v138, s4, v89, -v128
- ; GCN-NEXT: v_mul_f32_e32 v139, 0x3fb8aa3b, v139
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[126:127], v[0:15]
- ; GCN-NEXT: ; implicit-def: $sgpr0
- ; GCN-NEXT: v_perm_b32 v154, v135, v131, s5
- ; GCN-NEXT: v_perm_b32 v156, v135, v131, s7
- ; GCN-NEXT: v_fma_f32 v135, s4, v88, -v128
- ; GCN-NEXT: v_perm_b32 v150, v134, v130, s5
- ; GCN-NEXT: v_perm_b32 v152, v134, v130, s7
- ; GCN-NEXT: ds_read_b128 v[130:133], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v84
- ; GCN-NEXT: v_exp_f32_e32 v88, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v135, v85
- ; GCN-NEXT: v_perm_b32 v151, v146, v142, s5
- ; GCN-NEXT: v_perm_b32 v153, v146, v142, s7
- ; GCN-NEXT: v_perm_b32 v155, v147, v143, s5
- ; GCN-NEXT: v_perm_b32 v157, v147, v143, s7
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[148:149], v[126:127], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v89, v125
- ; GCN-NEXT: v_pack_b32_f16 v146, v134, v135
- ; GCN-NEXT: v_cvt_f16_f32_e32 v134, v86
- ; GCN-NEXT: v_fma_f32 v135, s4, v90, -v128
- ; GCN-NEXT: v_mul_f32_e32 v125, 0x3fb8aa3b, v138
- ; GCN-NEXT: v_mul_f32_e32 v148, 0x3fb8aa3b, v135
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[126:127], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v90, v158
- ; GCN-NEXT: v_mul_f32_e32 v158, 0x3fb8aa3b, v64
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[126:127], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v87
- ; GCN-NEXT: v_fma_f32 v127, s4, v91, -v128
- ; GCN-NEXT: v_exp_f32_e32 v91, v139
- ; GCN-NEXT: v_mul_f32_e32 v127, 0x3fb8aa3b, v127
- ; GCN-NEXT: v_pack_b32_f16 v147, v134, v126
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[130:131], v[146:147], v[0:15]
- ; GCN-NEXT: v_fma_f32 v130, s4, v92, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v88
- ; GCN-NEXT: v_exp_f32_e32 v92, v129
- ; GCN-NEXT: v_mul_f32_e32 v129, 0x3fb8aa3b, v130
- ; GCN-NEXT: v_cvt_f16_f32_e32 v130, v89
- ; GCN-NEXT: v_fma_f32 v131, s4, v93, -v128
- ; GCN-NEXT: v_pack_b32_f16 v130, v126, v130
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[142:143], v[146:147], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v93, v125
- ; GCN-NEXT: v_fma_f32 v126, s4, v94, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v125, v90
- ; GCN-NEXT: v_mul_f32_e32 v143, 0x3fb8aa3b, v126
- ; GCN-NEXT: v_cvt_f16_f32_e32 v126, v91
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_fma_f32 v131, s4, v95, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[134:135], v[146:147], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v94, v148
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v93
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[146:147], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v95, v127
- ; GCN-NEXT: v_cvt_f16_f32_e32 v127, v92
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v131
- ; GCN-NEXT: v_pack_b32_f16 v131, v125, v126
- ; GCN-NEXT: s_nop 1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[130:131], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v125, v129
- ; GCN-NEXT: ds_read_b128 v[132:135], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[146:149], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[144:145], v[130:131], v[32:47]
- ; GCN-NEXT: v_mul_f32_e32 v144, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_fma_f32 v65, s4, v66, -v128
- ; GCN-NEXT: v_exp_f32_e32 v126, v142
- ; GCN-NEXT: v_pack_b32_f16 v142, v127, v64
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v94
- ; GCN-NEXT: v_mul_f32_e32 v145, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v95
- ; GCN-NEXT: v_fma_f32 v66, s4, v67, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[136:137], v[130:131], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v127, v143
- ; GCN-NEXT: v_pack_b32_f16 v143, v64, v65
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[140:141], v[130:131], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v129, v138
- ; GCN-NEXT: v_mul_f32_e32 v141, 0x3fb8aa3b, v66
- ; GCN-NEXT: ds_read_b128 v[64:67], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[136:139], v197 offset:1728
+ ; GCN-NEXT: v_mul_f32_e32 v97, 0x3fb8aa3b, v85
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[110:111], v[104:105], v[48:63]
+ ; GCN-NEXT: v_mul_f32_e32 v110, 0x3fb8aa3b, v88
+ ; GCN-NEXT: v_add_f32_e32 v88, v118, v80
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v80, v120
+ ; GCN-NEXT: v_exp_f32_e32 v111, v121
+ ; GCN-NEXT: ds_read_b128 v[84:87], v139
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[100:103], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[82:83], v[104:105], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v82, v113
+ ; GCN-NEXT: v_exp_f32_e32 v113, v122
+ ; GCN-NEXT: v_pack_b32_f16 v80, v81, v80
+ ; GCN-NEXT: v_add_f32_e32 v81, v111, v88
+ ; GCN-NEXT: v_fma_f32 v89, s4, v89, -v148
+ ; GCN-NEXT: v_add_f32_e32 v88, v113, v81
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[98:99], v[104:105], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v81, v96
+ ; GCN-NEXT: v_exp_f32_e32 v104, v97
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v89
+ ; GCN-NEXT: v_fma_f32 v89, s4, v91, -v148
+ ; GCN-NEXT: v_fma_f32 v83, s4, v90, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v81, v82, v81
+ ; GCN-NEXT: v_mul_f32_e32 v82, 0x3fb8aa3b, v89
+ ; GCN-NEXT: v_add_f32_e32 v105, v104, v88
+ ; GCN-NEXT: ds_read_b128 v[88:91], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[96:99], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[84:85], v[80:81], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v110, v110
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v118
+ ; GCN-NEXT: v_fma_f32 v85, s4, v92, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v92, v111
+ ; GCN-NEXT: v_mul_f32_e32 v83, 0x3fb8aa3b, v83
+ ; GCN-NEXT: v_mul_f32_e32 v118, 0x3fb8aa3b, v85
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[100:101], v[80:81], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v111, v119
+ ; GCN-NEXT: v_add_f32_e32 v85, v110, v105
+ ; GCN-NEXT: v_pack_b32_f16 v92, v84, v92
+ ; GCN-NEXT: v_add_f32_e32 v84, v111, v85
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v85, v113
+ ; GCN-NEXT: v_fma_f32 v93, s4, v93, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[88:89], v[80:81], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v89, v83
+ ; GCN-NEXT: v_mul_f32_e32 v119, 0x3fb8aa3b, v93
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v110, v110
+ ; GCN-NEXT: v_fma_f32 v88, s4, v94, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v88, 0x3fb8aa3b, v88
+ ; GCN-NEXT: v_perm_b32 v100, v109, v107, s5
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[96:97], v[80:81], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v80, v104
+ ; GCN-NEXT: v_exp_f32_e32 v113, v82
+ ; GCN-NEXT: v_fma_f32 v81, s4, v95, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v93, v85, v80
+ ; GCN-NEXT: v_perm_b32 v104, v109, v107, s7
+ ; GCN-NEXT: v_add_f32_e32 v83, v89, v84
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[86:87], v[92:93], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v118, v118
+ ; GCN-NEXT: v_mul_f32_e32 v120, 0x3fb8aa3b, v81
+ ; GCN-NEXT: ds_read_b128 v[84:87], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v65, s4, v65, -v148
+ ; GCN-NEXT: v_fma_f32 v66, s4, v66, -v148
+ ; GCN-NEXT: v_add_f32_e32 v121, v113, v83
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[102:103], v[92:93], v[48:63]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v102, v111
+ ; GCN-NEXT: v_exp_f32_e32 v107, v119
+ ; GCN-NEXT: ds_read_b128 v[80:83], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_fma_f32 v64, s4, v64, -v148
+ ; GCN-NEXT: v_perm_b32 v94, v108, v106, s5
+ ; GCN-NEXT: v_perm_b32 v96, v108, v106, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[90:91], v[92:93], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v109, v88
+ ; GCN-NEXT: v_mul_f32_e32 v108, 0x3fb8aa3b, v65
+ ; GCN-NEXT: v_pack_b32_f16 v102, v110, v102
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v89
+ ; GCN-NEXT: v_mul_f32_e32 v110, 0x3fb8aa3b, v66
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v113
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[98:99], v[92:93], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v92, v120
+ ; GCN-NEXT: v_mul_f32_e32 v106, 0x3fb8aa3b, v64
+ ; GCN-NEXT: v_add_f32_e32 v64, v118, v121
+ ; GCN-NEXT: v_add_f32_e32 v64, v107, v64
+ ; GCN-NEXT: v_add_f32_e32 v64, v109, v64
+ ; GCN-NEXT: v_fma_f32 v67, s4, v67, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v103, v65, v66
+ ; GCN-NEXT: v_mul_f32_e32 v93, 0x3fb8aa3b, v67
+ ; GCN-NEXT: v_add_f32_e32 v98, v92, v64
+ ; GCN-NEXT: ds_read_b128 v[64:67], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_perm_b32 v95, v116, v114, s5
+ ; GCN-NEXT: v_perm_b32 v97, v116, v114, s7
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[84:85], v[102:103], v[32:47]
+ ; GCN-NEXT: ds_read_b128 v[88:91], v138 offset:1728
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
; GCN-NEXT: ;;#ASMSTART
@@ -950,211 +1004,164 @@
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
- ; GCN-NEXT: ds_write_b64 v199, v[150:151]
+ ; GCN-NEXT: ds_write_b64 v140, v[94:95]
+ ; GCN-NEXT: v_exp_f32_e32 v94, v106
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v200, v[152:153]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[132:133], v[142:143], v[0:15]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v132, v125
- ; GCN-NEXT: v_exp_f32_e32 v130, v158
+ ; GCN-NEXT: ds_write_b64 v141, v[96:97]
+ ; GCN-NEXT: v_fma_f32 v68, s4, v68, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v95, 0x3fb8aa3b, v68
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[80:81], v[102:103], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v96, v108
+ ; GCN-NEXT: v_add_f32_e32 v68, v94, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v118
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v80, v107
+ ; GCN-NEXT: v_fma_f32 v72, s4, v72, -v148
+ ; GCN-NEXT: v_fma_f32 v69, s4, v69, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[64:65], v[102:103], v[16:31]
+ ; GCN-NEXT: v_add_f32_e32 v64, v96, v68
+ ; GCN-NEXT: v_fma_f32 v68, s4, v70, -v148
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v109
+ ; GCN-NEXT: v_mul_f32_e32 v99, 0x3fb8aa3b, v68
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v92
+ ; GCN-NEXT: v_exp_f32_e32 v98, v110
+ ; GCN-NEXT: v_pack_b32_f16 v80, v84, v80
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[88:89], v[102:103], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v88, v93
+ ; GCN-NEXT: v_pack_b32_f16 v81, v65, v68
+ ; GCN-NEXT: v_add_f32_e32 v64, v98, v64
+ ; GCN-NEXT: v_add_f32_e32 v64, v88, v64
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v89, v94
+ ; GCN-NEXT: v_mul_f32_e32 v93, 0x3fb8aa3b, v72
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[86:87], v[80:81], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v92, v95
+ ; GCN-NEXT: v_mul_f32_e32 v97, 0x3fb8aa3b, v69
+ ; GCN-NEXT: v_perm_b32 v101, v117, v115, s5
+ ; GCN-NEXT: v_add_f32_e32 v72, v92, v64
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v96
+ ; GCN-NEXT: v_perm_b32 v105, v117, v115, s7
+ ; GCN-NEXT: v_fma_f32 v69, s4, v71, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[82:83], v[80:81], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v94, v97
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v201, v[154:155]
+ ; GCN-NEXT: ds_write_b64 v142, v[100:101]
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
- ; GCN-NEXT: ds_write_b64 v202, v[156:157]
+ ; GCN-NEXT: ds_write_b64 v143, v[104:105]
+ ; GCN-NEXT: v_mul_f32_e32 v65, 0x3fb8aa3b, v69
+ ; GCN-NEXT: v_fma_f32 v73, s4, v73, -v148
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[146:147], v[142:143], v[32:47]
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v126
- ; GCN-NEXT: v_exp_f32_e32 v131, v144
- ; GCN-NEXT: v_mul_f32_e32 v144, 0x3fb8aa3b, v69
- ; GCN-NEXT: v_fma_f32 v69, s4, v71, -v128
- ; GCN-NEXT: v_pack_b32_f16 v140, v132, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v129
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[64:65], v[142:143], v[16:31]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v127
- ; GCN-NEXT: v_exp_f32_e32 v132, v145
- ; GCN-NEXT: v_fma_f32 v65, s4, v70, -v128
- ; GCN-NEXT: v_mul_f32_e32 v65, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_fma_f32 v145, s4, v73, -v128
- ; GCN-NEXT: v_mul_f32_e32 v147, 0x3fb8aa3b, v145
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[136:137], v[142:143], v[48:63]
- ; GCN-NEXT: v_exp_f32_e32 v133, v141
- ; GCN-NEXT: v_mul_f32_e32 v142, 0x3fb8aa3b, v69
- ; GCN-NEXT: v_pack_b32_f16 v141, v64, v68
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[68:71], v198
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_fma_f32 v143, s4, v72, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v130
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[134:135], v[140:141], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v72, v146
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v143
- ; GCN-NEXT: v_cvt_f16_f32_e32 v143, v131
- ; GCN-NEXT: ds_read_b128 v[134:137], v198 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_pack_b32_f16 v64, v64, v143
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[148:149], v[140:141], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v73, v144
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[66:67], v[140:141], v[16:31]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v132
- ; GCN-NEXT: v_fma_f32 v67, s4, v74, -v128
- ; GCN-NEXT: v_exp_f32_e32 v74, v65
- ; GCN-NEXT: v_cvt_f16_f32_e32 v65, v133
- ; GCN-NEXT: v_mul_f32_e32 v67, 0x3fb8aa3b, v67
- ; GCN-NEXT: v_pack_b32_f16 v65, v66, v65
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[138:139], v[140:141], v[48:63]
- ; GCN-NEXT: v_fma_f32 v138, s4, v75, -v128
- ; GCN-NEXT: v_exp_f32_e32 v75, v142
- ; GCN-NEXT: v_mul_f32_e32 v148, 0x3fb8aa3b, v138
- ; GCN-NEXT: ds_read_b128 v[138:141], v198 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[142:145], v198 offset:1728
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v72
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[68:69], v[64:65], v[0:15]
- ; GCN-NEXT: v_fma_f32 v68, s4, v76, -v128
- ; GCN-NEXT: v_exp_f32_e32 v76, v146
- ; GCN-NEXT: v_mul_f32_e32 v146, 0x3fb8aa3b, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v73
- ; GCN-NEXT: v_fma_f32 v69, s4, v77, -v128
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[134:135], v[64:65], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v77, v147
- ; GCN-NEXT: v_pack_b32_f16 v134, v66, v68
- ; GCN-NEXT: v_fma_f32 v68, s4, v78, -v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v74
- ; GCN-NEXT: v_mul_f32_e32 v147, 0x3fb8aa3b, v69
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[138:139], v[64:65], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v78, v67
- ; GCN-NEXT: v_mul_f32_e32 v138, 0x3fb8aa3b, v68
- ; GCN-NEXT: v_cvt_f16_f32_e32 v139, v76
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[142:143], v[64:65], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v75
- ; GCN-NEXT: v_fma_f32 v65, s4, v79, -v128
- ; GCN-NEXT: v_exp_f32_e32 v79, v148
- ; GCN-NEXT: v_mul_f32_e32 v128, 0x3fb8aa3b, v65
- ; GCN-NEXT: v_pack_b32_f16 v135, v66, v64
- ; GCN-NEXT: s_nop 1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[70:71], v[134:135], v[0:15]
- ; GCN-NEXT: v_exp_f32_e32 v142, v146
- ; GCN-NEXT: ds_read_b128 v[68:71], v197
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: ds_read_b128 v[64:67], v197 offset:576
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[136:137], v[134:135], v[32:47]
- ; GCN-NEXT: v_exp_f32_e32 v137, v147
- ; GCN-NEXT: v_cvt_f16_f32_e32 v136, v77
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[140:141], v[134:135], v[16:31]
- ; GCN-NEXT: v_exp_f32_e32 v138, v138
- ; GCN-NEXT: v_cvt_f16_f32_e32 v140, v78
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[144:145], v[134:135], v[48:63]
- ; GCN-NEXT: s_nop 10
- ; GCN-NEXT: v_exp_f32_e32 v52, v128
- ; GCN-NEXT: v_cvt_f16_f32_e32 v50, v137
- ; GCN-NEXT: v_cvt_f16_f32_e32 v51, v142
- ; GCN-NEXT: v_cvt_f16_f32_e32 v54, v138
- ; GCN-NEXT: v_cvt_f16_f32_e32 v53, v52
- ; GCN-NEXT: v_cvt_f16_f32_e32 v49, v79
- ; GCN-NEXT: v_pack_b32_f16 v50, v51, v50
- ; GCN-NEXT: v_pack_b32_f16 v48, v139, v136
- ; GCN-NEXT: v_pack_b32_f16 v51, v54, v53
- ; GCN-NEXT: v_add_f32_e32 v53, 0, v113
- ; GCN-NEXT: v_add_f32_e32 v53, v114, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v115, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v116, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v117, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v118, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v119, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v120, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v121, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v122, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v123, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v124, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v96, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v97, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v98, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v99, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v100, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v101, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v102, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v103, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v104, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v105, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v106, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v107, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v108, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v109, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v110, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v111, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v80, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v81, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v82, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v83, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v84, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v85, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v86, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v87, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v88, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v89, v53
- ; GCN-NEXT: v_pack_b32_f16 v49, v140, v49
- ; GCN-NEXT: v_add_f32_e32 v53, v90, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v91, v53
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[68:69], v[48:49], v[0:15]
- ; GCN-NEXT: v_add_f32_e32 v53, v92, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v93, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v94, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v95, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v125, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v126, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v127, v53
- ; GCN-NEXT: v_add_f32_e32 v53, v129, v53
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[70:71], v[50:51], v[0:15]
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[64:65], v[48:49], v[32:47]
- ; GCN-NEXT: s_nop 9
- ; GCN-NEXT: v_add_f32_e32 v0, v130, v53
- ; GCN-NEXT: v_add_f32_e32 v0, v131, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v132, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v133, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v72, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v73, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v74, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v75, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v76, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v77, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v78, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v79, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v142, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v137, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v138, v0
- ; GCN-NEXT: v_add_f32_e32 v4, v52, v0
- ; GCN-NEXT: ds_bpermute_b32 v5, v196, v4
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: ds_read_b128 v[0:3], v197 offset:1152
- ; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[0:1], v[48:49], v[16:31]
- ; GCN-NEXT: v_add_f32_e32 v2, v4, v5
- ; GCN-NEXT: ds_bpermute_b32 v3, v196, v2
- ; GCN-NEXT: ; implicit-def: $vgpr4
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[66:67], v[80:81], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v95, v99
+ ; GCN-NEXT: v_pack_b32_f16 v64, v89, v64
+ ; GCN-NEXT: v_mul_f32_e32 v89, 0x3fb8aa3b, v73
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v98
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v73, v88
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[12:13]
- ; GCN-NEXT: v_fmac_f32_e32 v0, v4, v112
- ; GCN-NEXT: ds_read_b128 v[0:3], v197 offset:1728
+ ; GCN-NEXT: ds_read_b128 v[68:71], v139
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[90:91], v[80:81], v[0:15]
+ ; GCN-NEXT: v_exp_f32_e32 v88, v65
+ ; GCN-NEXT: v_add_f32_e32 v72, v94, v72
+ ; GCN-NEXT: v_fma_f32 v67, s4, v74, -v148
+ ; GCN-NEXT: v_add_f32_e32 v72, v95, v72
+ ; GCN-NEXT: v_fma_f32 v74, s4, v75, -v148
+ ; GCN-NEXT: ds_read_b128 v[84:87], v139 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_pack_b32_f16 v65, v66, v73
+ ; GCN-NEXT: v_mul_f32_e32 v66, 0x3fb8aa3b, v74
+ ; GCN-NEXT: v_add_f32_e32 v90, v88, v72
+ ; GCN-NEXT: ds_read_b128 v[72:75], v139 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[80:83], v139 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[68:69], v[64:65], v[32:47]
+ ; GCN-NEXT: v_mul_f32_e32 v67, 0x3fb8aa3b, v67
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v68, v92
+ ; GCN-NEXT: v_exp_f32_e32 v92, v93
+ ; GCN-NEXT: v_fma_f32 v69, s4, v76, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v69, 0x3fb8aa3b, v69
+ ; GCN-NEXT: v_fma_f32 v77, s4, v77, -v148
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[84:85], v[64:65], v[48:63]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v84, v94
+ ; GCN-NEXT: v_exp_f32_e32 v93, v89
+ ; GCN-NEXT: v_add_f32_e32 v76, v92, v90
+ ; GCN-NEXT: v_pack_b32_f16 v84, v68, v84
+ ; GCN-NEXT: v_mul_f32_e32 v94, 0x3fb8aa3b, v77
+ ; GCN-NEXT: v_add_f32_e32 v68, v93, v76
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[72:73], v[64:65], v[16:31]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v72, v95
+ ; GCN-NEXT: v_exp_f32_e32 v95, v67
+ ; GCN-NEXT: v_fma_f32 v73, s4, v78, -v148
+ ; GCN-NEXT: v_mul_f32_e32 v73, 0x3fb8aa3b, v73
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v92, v92
+ ; GCN-NEXT: v_add_f32_e32 v67, v95, v68
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[80:81], v[64:65], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v64, v88
+ ; GCN-NEXT: v_exp_f32_e32 v80, v66
+ ; GCN-NEXT: v_fma_f32 v65, s4, v79, -v148
+ ; GCN-NEXT: v_pack_b32_f16 v85, v72, v64
+ ; GCN-NEXT: v_mul_f32_e32 v81, 0x3fb8aa3b, v65
+ ; GCN-NEXT: v_add_f32_e32 v72, v80, v67
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[70:71], v[84:85], v[32:47]
+ ; GCN-NEXT: v_exp_f32_e32 v96, v69
+ ; GCN-NEXT: ds_read_b128 v[76:79], v138
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[88:91], v138 offset:576
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[68:71], v138 offset:1152
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: ds_read_b128 v[64:67], v138 offset:1728
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: buffer_inv sc0 sc1
+ ; GCN-NEXT: v_add_f32_e32 v67, v96, v72
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[86:87], v[84:85], v[48:63]
+ ; GCN-NEXT: v_exp_f32_e32 v72, v94
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v66, v93
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v86, v96
+ ; GCN-NEXT: v_add_f32_e32 v67, v72, v67
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v72, v72
+ ; GCN-NEXT: v_pack_b32_f16 v66, v92, v66
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[74:75], v[84:85], v[16:31]
+ ; GCN-NEXT: v_exp_f32_e32 v73, v73
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v74, v95
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[66:67], v[50:51], v[32:47]
+ ; GCN-NEXT: v_add_f32_e32 v75, v73, v67
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v67, v80
+ ; GCN-NEXT: v_exp_f32_e32 v80, v81
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[82:83], v[84:85], v[0:15]
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v73, v73
+ ; GCN-NEXT: v_pack_b32_f16 v67, v74, v67
+ ; GCN-NEXT: v_add_f32_e32 v74, v80, v75
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v75, v80
+ ; GCN-NEXT: v_pack_b32_f16 v72, v86, v72
+ ; GCN-NEXT: v_pack_b32_f16 v73, v73, v75
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[76:77], v[66:67], v[32:47]
+ ; GCN-NEXT: ds_bpermute_b32 v76, v136, v74
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_add_f32_e32 v74, v74, v76
+ ; GCN-NEXT: ds_bpermute_b32 v75, v136, v74
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[88:89], v[66:67], v[48:63]
+ ; GCN-NEXT: s_waitcnt lgkmcnt(0)
+ ; GCN-NEXT: v_cndmask_b32_e64 v74, v75, v74, s[12:13]
+ ; GCN-NEXT: v_fmac_f32_e32 v74, v137, v112
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[68:69], v[66:67], v[16:31]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[78:79], v[72:73], v[32:47]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[90:91], v[72:73], v[48:63]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[70:71], v[72:73], v[16:31]
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[64:65], v[66:67], v[0:15]
; GCN-NEXT: s_endpgm
attributes #0 = {"amdgpu-flat-work-group-size"="256,256"}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
index 2688a5fbaa889..5f95493c468d8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.exp.small.mir
@@ -66,7 +66,6 @@
; GCN-NEXT: ds_read_b128 v[44:47], v49 offset:512
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[36:37], v[40:41], 0
; GCN-NEXT: ; kill: killed $vgpr1
; GCN-NEXT: ; kill: killed $vgpr0
; GCN-NEXT: v_mul_lo_u32 v76, v76, s6
@@ -78,6 +77,7 @@
; GCN-NEXT: ; implicit-def: $sgpr3
; GCN-NEXT: v_lshl_add_u32 v79, v80, 1, v78
; GCN-NEXT: ; implicit-def: $sgpr0_sgpr1
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[36:37], v[40:41], 0
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[44:45], v[40:41], 0
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[16:31], v[38:39], v[42:43], v[16:31]
; GCN-NEXT: ds_read_b128 v[36:39], v51
@@ -148,13 +148,13 @@
; GCN-NEXT: buffer_wbl2 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: ds_write_b32 v78, v72
- ; GCN-NEXT: v_mul_f32_e32 v74, s4, v20
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[68:69], v[64:65], v[0:15]
; GCN-NEXT: v_mul_f32_e32 v64, s4, v16
; GCN-NEXT: v_mul_f32_e32 v65, s4, v17
; GCN-NEXT: v_mul_f32_e32 v68, s4, v18
; GCN-NEXT: v_mul_f32_e32 v69, s4, v19
; GCN-NEXT: v_max3_f32 v64, v64, s5, v65
+ ; GCN-NEXT: v_mul_f32_e32 v74, s4, v20
; GCN-NEXT: v_mul_f32_e32 v80, s4, v21
; GCN-NEXT: v_max3_f32 v64, v64, v68, v69
; GCN-NEXT: v_mul_f32_e32 v84, s4, v22
@@ -409,10 +409,10 @@
; GCN-NEXT: v_add_f32_e32 v0, v85, v17
; GCN-NEXT: v_cvt_f16_f32_e32 v17, v26
; GCN-NEXT: v_fma_f32 v12, s4, v12, -v72
- ; GCN-NEXT: v_exp_f32_e32 v8, v8
- ; GCN-NEXT: v_cvt_f16_f32_e32 v28, v67
+ ; GCN-NEXT: v_exp_f32_e32 v28, v8
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v8, v67
; GCN-NEXT: v_fma_f32 v13, s4, v13, -v72
- ; GCN-NEXT: v_exp_f32_e32 v9, v9
+ ; GCN-NEXT: v_exp_f32_e32 v29, v9
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[18:19], v[4:5], v[32:47]
; GCN-NEXT: v_add_f32_e32 v4, v88, v0
; GCN-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v10
@@ -420,78 +420,77 @@
; GCN-NEXT: v_fma_f32 v5, s4, v14, -v72
; GCN-NEXT: v_exp_f32_e32 v10, v0
; GCN-NEXT: v_cvt_f16_f32_e32 v0, v7
- ; GCN-NEXT: v_mul_f32_e32 v11, 0x3fb8aa3b, v11
- ; GCN-NEXT: v_fma_f32 v14, s4, v15, -v72
- ; GCN-NEXT: v_exp_f32_e32 v11, v11
- ; GCN-NEXT: v_pack_b32_f16 v1, v1, v0
- ; GCN-NEXT: v_pack_b32_f16 v0, v17, v28
+ ; GCN-NEXT: v_fma_f32 v9, s4, v15, -v72
; GCN-NEXT: ; implicit-def: $sgpr2
- ; GCN-NEXT: s_nop 1
+ ; GCN-NEXT: v_pack_b32_f16 v1, v1, v0
+ ; GCN-NEXT: v_pack_b32_f16 v0, v17, v8
+ ; GCN-NEXT: v_mul_f32_e32 v8, 0x3fb8aa3b, v11
+ ; GCN-NEXT: v_exp_f32_e32 v11, v8
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[2:3], v[0:1], v[48:63]
; GCN-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v12
; GCN-NEXT: v_add_f32_e32 v2, v30, v4
- ; GCN-NEXT: v_cvt_f16_f32_e32 v4, v8
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v4, v28
; GCN-NEXT: v_exp_f32_e32 v12, v3
; GCN-NEXT: v_mul_f32_e32 v3, 0x3fb8aa3b, v13
- ; GCN-NEXT: v_cvt_f16_f32_e32 v13, v9
- ; GCN-NEXT: v_exp_f32_e32 v15, v3
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v8, v29
+ ; GCN-NEXT: v_exp_f32_e32 v13, v3
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[20:21], v[0:1], v[32:47]
; GCN-NEXT: v_add_f32_e32 v0, v31, v2
; GCN-NEXT: v_add_f32_e32 v0, v22, v0
; GCN-NEXT: v_add_f32_e32 v0, v64, v0
; GCN-NEXT: v_add_f32_e32 v0, v23, v0
; GCN-NEXT: v_add_f32_e32 v0, v25, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v27, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v65, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v68, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v24, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v26, v0
- ; GCN-NEXT: v_add_f32_e32 v0, v67, v0
; GCN-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v5
- ; GCN-NEXT: v_add_f32_e32 v0, v6, v0
+ ; GCN-NEXT: v_add_f32_e32 v0, v27, v0
; GCN-NEXT: v_cvt_f16_f32_e32 v5, v10
- ; GCN-NEXT: v_exp_f32_e32 v17, v1
- ; GCN-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v14
- ; GCN-NEXT: v_add_f32_e32 v0, v7, v0
- ; GCN-NEXT: v_cvt_f16_f32_e32 v14, v11
- ; GCN-NEXT: v_exp_f32_e32 v18, v1
- ; GCN-NEXT: v_add_f32_e32 v6, v8, v0
+ ; GCN-NEXT: v_exp_f32_e32 v14, v1
+ ; GCN-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v9
+ ; GCN-NEXT: v_add_f32_e32 v0, v65, v0
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v9, v11
+ ; GCN-NEXT: v_exp_f32_e32 v15, v1
+ ; GCN-NEXT: v_add_f32_e32 v17, v68, v0
; GCN-NEXT: ds_read_b128 v[0:3], v91
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_add_f32_e32 v6, v9, v6
- ; GCN-NEXT: v_pack_b32_f16 v9, v5, v14
- ; GCN-NEXT: v_pack_b32_f16 v8, v4, v13
- ; GCN-NEXT: v_add_f32_e32 v6, v10, v6
- ; GCN-NEXT: v_cvt_f16_f32_e32 v7, v18
- ; GCN-NEXT: v_cvt_f16_f32_e32 v10, v15
+ ; GCN-NEXT: v_pack_b32_f16 v9, v5, v9
+ ; GCN-NEXT: v_pack_b32_f16 v8, v4, v8
+ ; GCN-NEXT: v_add_f32_e32 v17, v24, v17
+ ; GCN-NEXT: v_add_f32_e32 v17, v26, v17
+ ; GCN-NEXT: v_add_f32_e32 v17, v67, v17
+ ; GCN-NEXT: v_add_f32_e32 v6, v6, v17
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v17, v15
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v18, v13
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[0:1], v[8:9], v[48:63]
- ; GCN-NEXT: v_cvt_f16_f32_e32 v0, v17
+ ; GCN-NEXT: v_cvt_f16_f32_e32 v0, v14
; GCN-NEXT: v_cvt_f16_f32_e32 v4, v12
- ; GCN-NEXT: v_add_f32_e32 v6, v11, v6
- ; GCN-NEXT: v_add_f32_e32 v6, v12, v6
- ; GCN-NEXT: v_add_f32_e32 v1, v15, v6
- ; GCN-NEXT: v_add_f32_e32 v11, v17, v1
- ; GCN-NEXT: v_pack_b32_f16 v1, v0, v7
- ; GCN-NEXT: v_pack_b32_f16 v0, v4, v10
+ ; GCN-NEXT: v_add_f32_e32 v1, v7, v6
+ ; GCN-NEXT: v_add_f32_e32 v19, v28, v1
+ ; GCN-NEXT: v_pack_b32_f16 v1, v0, v17
+ ; GCN-NEXT: v_pack_b32_f16 v0, v4, v18
; GCN-NEXT: ds_read_b128 v[4:7], v91 offset:576
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_inv sc0 sc1
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[4:5], v[8:9], v[32:47]
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: ;;#ASMEND
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[4:5], v[8:9], v[32:47]
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[48:63], v[2:3], v[0:1], v[48:63]
- ; GCN-NEXT: v_add_f32_e32 v2, v18, v11
+ ; GCN-NEXT: v_add_f32_e32 v2, v29, v19
+ ; GCN-NEXT: v_add_f32_e32 v2, v10, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v11, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v12, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v13, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v14, v2
+ ; GCN-NEXT: v_add_f32_e32 v2, v15, v2
; GCN-NEXT: ds_bpermute_b32 v3, v66, v2
+ ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[6:7], v[0:1], v[32:47]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_f32_e32 v2, v2, v3
; GCN-NEXT: ds_bpermute_b32 v3, v66, v2
- ; GCN-NEXT: v_mfma_f32_32x32x8_f16 v[32:47], v[6:7], v[0:1], v[32:47]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
- ; GCN-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[0:1]
- ; GCN-NEXT: v_fmac_f32_e32 v2, v4, v16
+ ; GCN-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
+ ; GCN-NEXT: v_fmac_f32_e32 v0, v4, v16
; GCN-NEXT: s_endpgm
attributes #0 = {"amdgpu-flat-work-group-size"="256,256"}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
index 279de9b7899e0..5eb1e144614a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
@@ -902,8 +902,7 @@ define amdgpu_kernel void @test_mfma_f32_16x16x8bf16(ptr addrspace(1) %arg) #0 {
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v2, 2
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NEXT: v_mov_b32_e32 v1, 2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
@@ -912,26 +911,27 @@ define amdgpu_kernel void @test_mfma_f32_16x16x8bf16(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
+; GFX90A-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: s_nop 9
+; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x8bf16:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 2
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x8bf16 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: s_nop 10
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x8bf16 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-NEXT: s_nop 9
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index ce660cd4bb6c3..1719e966fef97 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -82,56 +82,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[0:1], 48
+; SDAG-AGPR-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a31, s23
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a30, s22
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a29, s21
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a28, s20
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a27, s19
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a26, s18
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a25, s17
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a24, s16
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a23, s15
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a22, s14
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a21, s13
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a20, s12
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a19, s11
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a18, s10
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a17, s9
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a16, s8
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[2:5], v[6:9], a[16:31]
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s17
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], 32
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[4:5], 16
-; SDAG-AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a0, s8
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a1, s9
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a2, s10
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a3, s11
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a4, s12
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a5, s13
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a6, s14
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a7, s15
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a8, s16
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a9, s17
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a10, s18
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a11, s19
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a12, s20
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a13, s21
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[8:11], v[12:15], a[0:15]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s21
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s23
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: s_nop 4
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; SDAG-AGPR-NEXT: s_nop 6
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s20
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s21
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s22
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s23
; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: v_mov_b32_e32 v0, s8
; SDAG-AGPR-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s11
; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
@@ -155,43 +155,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a0, s8
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a1, s9
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a2, s10
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a3, s11
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a4, s12
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a5, s13
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a6, s14
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a7, s15
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a8, s16
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a9, s17
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a10, s18
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a11, s19
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a12, s20
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a13, s21
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a31, s23
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a30, s22
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a29, s21
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a28, s20
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a27, s19
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a26, s18
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a25, s17
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a24, s16
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a23, s15
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a22, s14
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a21, s13
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a20, s12
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a19, s11
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a18, s10
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a17, s9
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a16, s8
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], 48
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15]
+; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[16:31]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-AGPR-NEXT: s_nop 10
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v0, a16
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v1, a17
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v2, a18
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v3, a19
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v4, a20
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v5, a21
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v6, a22
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v7, a23
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v8, a24
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v9, a25
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v10, a26
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v11, a27
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v12, a28
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v13, a29
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v14, a30
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v15, a31
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v15, a15
; GISEL-AGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: global_store_dwordx4 v[22:23], v[4:7], off sc0 sc1
@@ -223,37 +223,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v48, s16
-; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-VGPR-NEXT: s_nop 8
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-VGPR-NEXT: s_nop 1
+; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
+; SDAG-VGPR-NEXT: s_nop 6
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], 32
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], 16
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s19
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s20
; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s21
@@ -266,14 +267,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s10
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
;
@@ -281,50 +282,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
; GISEL-VGPR: ; %bb.0:
; GISEL-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-VGPR-NEXT: s_nop 10
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-VGPR-NEXT: s_nop 1
+; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-VGPR-NEXT: s_nop 6
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
@@ -405,56 +407,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[0:1], 48
+; SDAG-AGPR-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a31, s23
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a30, s22
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a29, s21
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a28, s20
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a27, s19
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a26, s18
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a25, s17
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a24, s16
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a23, s15
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a22, s14
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a21, s13
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a20, s12
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a19, s11
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a18, s10
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a17, s9
+; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a16, s8
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[2:5], v[6:9], a[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s17
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[2:3], 32
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[4:5], 16
-; SDAG-AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a0, s8
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a1, s9
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a2, s10
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a3, s11
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a4, s12
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a5, s13
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a6, s14
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a7, s15
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a8, s16
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a9, s17
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a10, s18
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a11, s19
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a12, s20
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a13, s21
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
-; SDAG-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[6:7], 0
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s21
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s23
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: s_nop 4
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; SDAG-AGPR-NEXT: s_nop 6
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v8, s20
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v9, s21
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v10, s22
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v11, s23
; SDAG-AGPR-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: v_mov_b32_e32 v0, s8
; SDAG-AGPR-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v3, s11
; SDAG-AGPR-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
@@ -478,43 +480,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a0, s8
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a1, s9
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a2, s10
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a3, s11
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a4, s12
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a5, s13
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a6, s14
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a7, s15
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a8, s16
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a9, s17
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a10, s18
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a11, s19
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a12, s20
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a13, s21
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a14, s22
-; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a15, s23
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a31, s23
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a30, s22
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a29, s21
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a28, s20
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a27, s19
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a26, s18
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a25, s17
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a24, s16
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a23, s15
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a22, s14
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a21, s13
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a20, s12
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a19, s11
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a18, s10
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a17, s9
+; GISEL-AGPR-NEXT: v_accvgpr_write_b32 a16, s8
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], 48
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
+; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:2 abid:3 blgp:1
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-AGPR-NEXT: s_nop 10
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v0, a16
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v1, a17
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v2, a18
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v3, a19
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v4, a20
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v5, a21
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v6, a22
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v7, a23
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v8, a24
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v9, a25
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v10, a26
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v11, a27
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v12, a28
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v13, a29
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v14, a30
-; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v15, a31
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-AGPR-NEXT: v_accvgpr_read_b32 v15, a15
; GISEL-AGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: global_store_dwordx4 v[22:23], v[4:7], off sc0 sc1
@@ -546,37 +548,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v48, s16
-; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-VGPR-NEXT: s_nop 8
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-VGPR-NEXT: s_nop 1
+; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-VGPR-NEXT: s_nop 6
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], 32
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], 16
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s19
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s20
; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s21
@@ -589,14 +592,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s10
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-VGPR-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
;
@@ -604,50 +607,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
; GISEL-VGPR: ; %bb.0:
; GISEL-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-VGPR-NEXT: s_nop 10
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-VGPR-NEXT: s_nop 1
+; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; GISEL-VGPR-NEXT: s_nop 6
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 2, i32 3, i32 1)
@@ -990,44 +994,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; GISEL-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-AGPR-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-AGPR-NEXT: v_mov_b32_e32 v44, 0
; GISEL-AGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-AGPR-NEXT: s_nop 6
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_endpgm
;
@@ -1093,44 +1098,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; GISEL-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v44, 0
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-VGPR-NEXT: s_nop 6
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
@@ -1259,44 +1265,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; GISEL-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-AGPR-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-AGPR-NEXT: v_mov_b32_e32 v44, 0
; GISEL-AGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-AGPR-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; GISEL-AGPR-NEXT: s_nop 6
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-AGPR-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-AGPR-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-AGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-AGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-AGPR-NEXT: s_endpgm
;
@@ -1362,44 +1369,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; GISEL-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v44, 0
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; GISEL-VGPR-NEXT: s_nop 6
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
-; GISEL-VGPR-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 1, i32 2, i32 3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 42cd877c42261..492ac9ded158c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -363,37 +363,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: v_mov_b32_e32 v48, s16
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-NEXT: s_nop 8
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 32
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 16
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 0
+; SDAG-NEXT: v_mov_b32_e32 v22, s16
+; SDAG-NEXT: v_mov_b32_e32 v23, s17
+; SDAG-NEXT: v_mov_b32_e32 v24, s18
+; SDAG-NEXT: v_mov_b32_e32 v25, s19
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
@@ -406,14 +407,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -421,50 +422,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -473,56 +475,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT: v_mov_b32_e32 v10, s18
+; HEURRC-NEXT: v_mov_b32_e32 v11, s19
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[2:5], v[6:9], a[16:31]
+; HEURRC-NEXT: v_mov_b32_e32 v8, s16
+; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15]
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: s_nop 4
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT: s_nop 6
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v8, s20
+; HEURRC-NEXT: v_mov_b32_e32 v9, s21
+; HEURRC-NEXT: v_mov_b32_e32 v10, s22
+; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -539,37 +541,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]
-; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
-; VGPRRC-NEXT: s_nop 8
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 32
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v24, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v25, s19
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
@@ -582,14 +585,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16:
@@ -729,37 +732,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
-; SDAG-NEXT: v_mov_b32_e32 v48, s16
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-NEXT: v_mov_b32_e32 v49, s17
-; SDAG-NEXT: v_mov_b32_e32 v50, s18
-; SDAG-NEXT: v_mov_b32_e32 v51, s19
-; SDAG-NEXT: s_nop 8
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 32
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 16
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 0
+; SDAG-NEXT: v_mov_b32_e32 v22, s16
+; SDAG-NEXT: v_mov_b32_e32 v23, s17
+; SDAG-NEXT: v_mov_b32_e32 v24, s18
+; SDAG-NEXT: v_mov_b32_e32 v25, s19
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
@@ -772,14 +776,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -787,50 +791,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -839,56 +844,56 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[30:31]
+; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT: v_mov_b32_e32 v10, s18
+; HEURRC-NEXT: v_mov_b32_e32 v11, s19
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[2:5], v[6:9], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mov_b32_e32 v8, s16
+; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
-; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
-; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
-; HEURRC-NEXT: v_mov_b32_e32 v8, s20
-; HEURRC-NEXT: v_mov_b32_e32 v9, s21
-; HEURRC-NEXT: v_mov_b32_e32 v10, s22
-; HEURRC-NEXT: v_mov_b32_e32 v11, s23
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: s_nop 4
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT: s_nop 6
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v8, s20
+; HEURRC-NEXT: v_mov_b32_e32 v9, s21
+; HEURRC-NEXT: v_mov_b32_e32 v10, s22
+; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
-; HEURRC-NEXT: v_mov_b32_e32 v2, s10
-; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -905,37 +910,38 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]
-; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
-; VGPRRC-NEXT: s_nop 8
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 32
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v24, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v25, s19
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[22:25], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
@@ -948,14 +954,14 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__flags:
@@ -1458,44 +1464,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -1800,44 +1807,45 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -2856,64 +2864,66 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v40, s24
-; SDAG-NEXT: v_mov_b32_e32 v41, s25
-; SDAG-NEXT: v_mov_b32_e32 v42, s26
-; SDAG-NEXT: v_mov_b32_e32 v43, s27
-; SDAG-NEXT: v_mov_b32_e32 v44, s28
-; SDAG-NEXT: v_mov_b32_e32 v45, s29
-; SDAG-NEXT: v_mov_b32_e32 v46, s30
-; SDAG-NEXT: v_mov_b32_e32 v47, s31
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
+; SDAG-NEXT: v_mov_b32_e32 v32, s24
+; SDAG-NEXT: v_mov_b32_e32 v33, s25
+; SDAG-NEXT: v_mov_b32_e32 v34, s26
+; SDAG-NEXT: v_mov_b32_e32 v35, s27
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s16
; SDAG-NEXT: v_mov_b32_e32 v1, s17
; SDAG-NEXT: v_mov_b32_e32 v2, s18
; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -2921,50 +2931,51 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -2972,49 +2983,48 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
-; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
-; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v8, s24
-; HEURRC-NEXT: v_mov_b32_e32 v9, s25
-; HEURRC-NEXT: v_mov_b32_e32 v10, s26
-; HEURRC-NEXT: v_mov_b32_e32 v11, s27
-; HEURRC-NEXT: v_mov_b32_e32 v12, s28
-; HEURRC-NEXT: v_mov_b32_e32 v13, s29
-; HEURRC-NEXT: v_mov_b32_e32 v14, s30
-; HEURRC-NEXT: v_mov_b32_e32 v15, s31
-; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
-; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15]
+; HEURRC-NEXT: v_mov_b32_e32 v0, s24
+; HEURRC-NEXT: v_mov_b32_e32 v1, s25
+; HEURRC-NEXT: v_mov_b32_e32 v2, s26
+; HEURRC-NEXT: v_mov_b32_e32 v3, s27
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
+; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
-; HEURRC-NEXT: s_nop 7
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
+; HEURRC-NEXT: s_nop 5
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
@@ -3043,64 +3053,66 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v47, s31
-; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
+; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 48
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 32
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], 0
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
-; VGPRRC-NEXT: s_nop 11
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8:
@@ -3251,64 +3263,66 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v40, s24
-; SDAG-NEXT: v_mov_b32_e32 v41, s25
-; SDAG-NEXT: v_mov_b32_e32 v42, s26
-; SDAG-NEXT: v_mov_b32_e32 v43, s27
-; SDAG-NEXT: v_mov_b32_e32 v44, s28
-; SDAG-NEXT: v_mov_b32_e32 v45, s29
-; SDAG-NEXT: v_mov_b32_e32 v46, s30
-; SDAG-NEXT: v_mov_b32_e32 v47, s31
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
+; SDAG-NEXT: v_mov_b32_e32 v32, s24
+; SDAG-NEXT: v_mov_b32_e32 v33, s25
+; SDAG-NEXT: v_mov_b32_e32 v34, s26
+; SDAG-NEXT: v_mov_b32_e32 v35, s27
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
+; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: s_nop 6
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s16
; SDAG-NEXT: v_mov_b32_e32 v1, s17
; SDAG-NEXT: v_mov_b32_e32 v2, s18
; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -3316,50 +3330,51 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], 16
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], 32
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], 48
-; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: s_nop 10
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], 16
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], 32
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], 48
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[44:45], v[40:43], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[40:41], v[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
-; GISEL-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
-; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[24:25], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -3367,49 +3382,48 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
-; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
-; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v8, s24
-; HEURRC-NEXT: v_mov_b32_e32 v9, s25
-; HEURRC-NEXT: v_mov_b32_e32 v10, s26
-; HEURRC-NEXT: v_mov_b32_e32 v11, s27
-; HEURRC-NEXT: v_mov_b32_e32 v12, s28
-; HEURRC-NEXT: v_mov_b32_e32 v13, s29
-; HEURRC-NEXT: v_mov_b32_e32 v14, s30
-; HEURRC-NEXT: v_mov_b32_e32 v15, s31
-; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
-; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mov_b32_e32 v0, s24
+; HEURRC-NEXT: v_mov_b32_e32 v1, s25
+; HEURRC-NEXT: v_mov_b32_e32 v2, s26
+; HEURRC-NEXT: v_mov_b32_e32 v3, s27
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
+; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
+; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
-; HEURRC-NEXT: s_nop 7
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
+; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
+; HEURRC-NEXT: s_nop 5
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
@@ -3438,64 +3452,66 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v47, s31
-; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
+; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: s_nop 1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: s_nop 6
+; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], 48
+; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], 32
+; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], 0
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
-; VGPRRC-NEXT: s_nop 11
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[20:21], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:
@@ -4017,44 +4033,45 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -4394,44 +4411,45 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; GISEL-NEXT: v_mov_b32_e32 v56, 0
+; GISEL-NEXT: v_mov_b32_e32 v44, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
-; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
-; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
-; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
-; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
-; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
-; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
-; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
-; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
+; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; GISEL-NEXT: s_nop 6
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[20:23], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[24:27], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -4704,19 +4722,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v20, s24
-; SDAG-NEXT: v_mov_b32_e32 v21, s25
-; SDAG-NEXT: v_mov_b32_e32 v22, s26
-; SDAG-NEXT: v_mov_b32_e32 v23, s27
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s24
+; SDAG-NEXT: v_mov_b32_e32 v17, s25
+; SDAG-NEXT: v_mov_b32_e32 v18, s26
+; SDAG-NEXT: v_mov_b32_e32 v19, s27
+; SDAG-NEXT: v_mov_b32_e32 v20, s28
+; SDAG-NEXT: v_mov_b32_e32 v21, s29
+; SDAG-NEXT: v_mov_b32_e32 v22, s30
+; SDAG-NEXT: v_mov_b32_e32 v23, s31
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4765,19 +4782,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v20, s24
-; HEURRC-NEXT: v_mov_b32_e32 v21, s25
-; HEURRC-NEXT: v_mov_b32_e32 v22, s26
-; HEURRC-NEXT: v_mov_b32_e32 v23, s27
+; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v16, s24
+; HEURRC-NEXT: v_mov_b32_e32 v17, s25
+; HEURRC-NEXT: v_mov_b32_e32 v18, s26
+; HEURRC-NEXT: v_mov_b32_e32 v19, s27
+; HEURRC-NEXT: v_mov_b32_e32 v20, s28
+; HEURRC-NEXT: v_mov_b32_e32 v21, s29
+; HEURRC-NEXT: v_mov_b32_e32 v22, s30
+; HEURRC-NEXT: v_mov_b32_e32 v23, s31
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4798,19 +4814,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
+; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v20, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v21, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4910,19 +4925,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v20, s24
-; SDAG-NEXT: v_mov_b32_e32 v21, s25
-; SDAG-NEXT: v_mov_b32_e32 v22, s26
-; SDAG-NEXT: v_mov_b32_e32 v23, s27
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s24
+; SDAG-NEXT: v_mov_b32_e32 v17, s25
+; SDAG-NEXT: v_mov_b32_e32 v18, s26
+; SDAG-NEXT: v_mov_b32_e32 v19, s27
+; SDAG-NEXT: v_mov_b32_e32 v20, s28
+; SDAG-NEXT: v_mov_b32_e32 v21, s29
+; SDAG-NEXT: v_mov_b32_e32 v22, s30
+; SDAG-NEXT: v_mov_b32_e32 v23, s31
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4971,19 +4985,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v20, s24
-; HEURRC-NEXT: v_mov_b32_e32 v21, s25
-; HEURRC-NEXT: v_mov_b32_e32 v22, s26
-; HEURRC-NEXT: v_mov_b32_e32 v23, s27
+; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v16, s24
+; HEURRC-NEXT: v_mov_b32_e32 v17, s25
+; HEURRC-NEXT: v_mov_b32_e32 v18, s26
+; HEURRC-NEXT: v_mov_b32_e32 v19, s27
+; HEURRC-NEXT: v_mov_b32_e32 v20, s28
+; HEURRC-NEXT: v_mov_b32_e32 v21, s29
+; HEURRC-NEXT: v_mov_b32_e32 v22, s30
+; HEURRC-NEXT: v_mov_b32_e32 v23, s31
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -5004,19 +5017,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
-; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
+; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v20, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v21, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v22, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v23, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 49d4662300f76..9f78dd4d40bfc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -1,12 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.0a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GFX90A-GISEL %s
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.0a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a < %s | FileCheck --check-prefixes=GFX90A-VGPR-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR %s
declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32, i32, <16 x i32>, i32, i32, i32)
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32, i32, <4 x i32>, i32, i32, i32)
@@ -147,145 +145,67 @@ define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX908-GISEL-NEXT: s_endpgm
;
-; GFX90A-SDAG-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A-SDAG: ; %bb.0: ; %bb
-; GFX90A-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-SDAG-NEXT: v_mov_b32_e32 v1, 2
-; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a4, s4
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a5, s5
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a6, s6
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a7, s7
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a8, s8
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a9, s9
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a10, s10
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a11, s11
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a12, s12
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a13, s13
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a14, s14
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a15, s15
-; GFX90A-SDAG-NEXT: s_nop 1
-; GFX90A-SDAG-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-SDAG-NEXT: s_nop 15
-; GFX90A-SDAG-NEXT: s_nop 1
-; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
-; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
-; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
-; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
-; GFX90A-SDAG-NEXT: s_endpgm
+; GFX90A-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A: ; %bb.0: ; %bb
+; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4
+; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5
+; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6
+; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7
+; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8
+; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9
+; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10
+; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11
+; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12
+; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13
+; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14
+; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: s_nop 15
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
+; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
+; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
+; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
+; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a4, s4
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a5, s5
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a6, s6
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a7, s7
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a8, s8
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a9, s9
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a10, s10
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a11, s11
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a12, s12
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a13, s13
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a14, s14
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a15, s15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
-; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A-VGPR-SDAG: ; %bb.0: ; %bb
-; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: s_nop 1
-; GFX90A-VGPR-SDAG-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-VGPR-SDAG-NEXT: s_nop 15
-; GFX90A-VGPR-SDAG-NEXT: s_nop 1
-; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-VGPR-SDAG-NEXT: s_endpgm
-;
-; GFX90A-VGPR-GISEL-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: s_nop 1
-; GFX90A-VGPR-GISEL-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-VGPR-GISEL-NEXT: s_nop 15
-; GFX90A-VGPR-GISEL-NEXT: s_nop 1
-; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+; GFX90A-VGPR-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-VGPR: ; %bb.0: ; %bb
+; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: s_nop 1
+; GFX90A-VGPR-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-NEXT: s_nop 15
+; GFX90A-VGPR-NEXT: s_nop 1
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -322,77 +242,41 @@ define amdgpu_kernel void @test_mfma_i32_16x16x16i8(ptr addrspace(1) %arg) #0 {
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX908-NEXT: s_endpgm
;
-; GFX90A-SDAG-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A-SDAG: ; %bb.0: ; %bb
-; GFX90A-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-SDAG-NEXT: v_mov_b32_e32 v2, 2
-; GFX90A-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-SDAG-NEXT: s_nop 1
-; GFX90A-SDAG-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-SDAG-NEXT: s_nop 10
-; GFX90A-SDAG-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
-; GFX90A-SDAG-NEXT: s_endpgm
-;
-; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A-VGPR-SDAG: ; %bb.0: ; %bb
-; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v6, 2
-; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-SDAG-NEXT: s_nop 1
-; GFX90A-VGPR-SDAG-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-SDAG-NEXT: s_nop 10
-; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
-; GFX90A-VGPR-SDAG-NEXT: s_endpgm
+; GFX90A-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A: ; %bb.0: ; %bb
+; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-NEXT: s_nop 1
+; GFX90A-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: s_nop 9
+; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
+; GFX90A-NEXT: s_endpgm
;
-; GFX90A-VGPR-GISEL-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-GISEL-NEXT: s_nop 1
-; GFX90A-VGPR-GISEL-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-VGPR-GISEL-NEXT: s_nop 9
-; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+; GFX90A-VGPR-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-VGPR: ; %bb.0: ; %bb
+; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-NEXT: s_nop 1
+; GFX90A-VGPR-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-NEXT: s_nop 9
+; GFX90A-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 086fc6f0ed4d7..a102020474a87 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -1,14 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgpu9.08 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC,GFX908,GFX908_A %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgpu9.08 < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-NOLIT-SRCC %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgpu9.08 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgpu9.08 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-LIT-SRCC %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgpu9.0a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgpu9.0a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-GISEL %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgpu9.42 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgpu9.42 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-GISEL %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgpu9.42 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgpu9.42 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC,GFX908,GFX908_A %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-NOLIT-SRCC %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-LIT-SRCC %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32)
@@ -472,46 +469,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -568,46 +525,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -663,46 +580,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 1, i32 2, i32 3)
@@ -937,32 +814,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -989,32 +840,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_nop 8
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1040,32 +865,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 8
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1175,23 +974,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_nop 3
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1209,23 +991,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_nop 2
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1242,23 +1007,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 2
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1495,33 +1243,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x2f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x2f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x2f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1549,33 +1270,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x2f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x2f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1602,33 +1296,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x2f32:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1725,103 +1392,52 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 2.0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: s_nop 9
+; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x4f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 2.0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-NEXT: s_nop 9
-; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: s_nop 8
+; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_nop 8
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 2.0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT: s_nop 1
-; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-NEXT: s_nop 9
-; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-NEXT: s_nop 8
+; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4f32:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 8
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x4f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3)
@@ -2291,47 +1907,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x4f16:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[40:41], s[40:41] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[42:43], s[42:43] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x4f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -2392,47 +1967,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x4f16:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[40:41]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[42:43]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -2492,47 +2026,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4f16:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[40:41]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[42:43]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2782,33 +2275,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4f16:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2838,33 +2304,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4f16:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2893,33 +2332,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4f16:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 9
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -3044,24 +2456,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x4f16:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x4f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_nop 3
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -3082,24 +2476,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x4f16:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_nop 3
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -3119,24 +2495,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4f16:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 3
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -3388,34 +2746,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8f16:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x8f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -3445,33 +2775,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8f16:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -3500,33 +2803,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8f16:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 9
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -3541,7 +2817,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; NOLIT-SRCC-LABEL: test_mfma_f32_16x16x16f16:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; NOLIT-SRCC-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
@@ -3549,16 +2824,17 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s4
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s8
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s5
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s9
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s9
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v5
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v7, s10
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s10
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s11
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v6
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v7
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v6
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v5
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s6
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; NOLIT-SRCC-NEXT: s_nop 1
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; NOLIT-SRCC-NEXT: s_nop 0
; NOLIT-SRCC-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 9
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
@@ -3603,7 +2879,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; LIT-SRCC-LABEL: test_mfma_f32_16x16x16f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
; LIT-SRCC-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; LIT-SRCC-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
@@ -3611,16 +2886,17 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s4
; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s8
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s5
-; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s9
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s9
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v5
-; LIT-SRCC-NEXT: v_mov_b32_e32 v7, s10
+; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s10
; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s11
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v6
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v7
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v6
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v5
; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s6
; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s7
-; LIT-SRCC-NEXT: s_nop 1
+; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; LIT-SRCC-NEXT: s_nop 0
; LIT-SRCC-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 9
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
@@ -3634,41 +2910,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX90A-LABEL: test_mfma_f32_16x16x16f16:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v6, s4
-; GFX90A-NEXT: v_mov_b32_e32 v7, s5
-; GFX90A-NEXT: v_mov_b32_e32 v8, s6
-; GFX90A-NEXT: v_mov_b32_e32 v9, s7
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: v_mov_b32_e32 v5, s5
+; GFX90A-NEXT: v_mov_b32_e32 v6, s6
+; GFX90A-NEXT: v_mov_b32_e32 v7, s7
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[6:7], v[8:9], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
+; GFX90A-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: s_nop 9
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x16f16:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x16f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -3689,24 +2947,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x16f16:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_nop 5
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -3726,24 +2966,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-VGPR-NEXT: s_nop 6
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16f16:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 5
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -4173,46 +3395,6 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_32x32x4i8:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x4i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_i32_32x32x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -4269,46 +3451,6 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_i32_32x32x4i8:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_i32_32x32x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -4364,46 +3506,6 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_32x32x4i8:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 1, i32 2, <32 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -4638,32 +3740,6 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_i32_16x16x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -4690,32 +3766,6 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -4741,32 +3791,6 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 9
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -4947,46 +3971,6 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-GISEL-NEXT: s_mov_b32 s8, 64
-; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 8
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 1
@@ -5003,46 +3987,6 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX942-GISEL-NEXT: s_mov_b32 s8, 64
-; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 8
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1
@@ -5058,46 +4002,6 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 64
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 8
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> splat (i32 64), i32 1, i32 2, i32 3)
@@ -5207,23 +4111,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_nop 3
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_i32_4x4x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -5241,23 +4128,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_nop 3
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -5274,23 +4144,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 3
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -5392,25 +4245,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1
@@ -5424,25 +4258,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 1
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 2
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1
@@ -5455,25 +4270,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 2
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 1), i32 1, i32 2, i32 3)
@@ -5572,25 +4368,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
-; GFX90A-GISEL: ; %bb.0:
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-GISEL-NEXT: s_movk_i32 s0, 0x41
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; GFX942: ; %bb.0:
; GFX942-NEXT: v_mov_b32_e32 v5, 1
@@ -5608,25 +4385,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
-; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-GISEL-NEXT: s_movk_i32 s0, 0x41
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 2
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; GFX942-VGPR: ; %bb.0:
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1
@@ -5643,25 +4401,6 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
-; GFX942-VGPR-GISEL: ; %bb.0:
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
-; GFX942-VGPR-GISEL-NEXT: s_movk_i32 s0, 0x41
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 2
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 65), i32 1, i32 2, i32 3)
store <4 x i32> %mai.1, ptr addrspace(1) %arg
@@ -6119,47 +4858,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -6217,47 +4915,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -6314,47 +4971,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
@@ -6592,33 +5208,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_nop 9
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -6646,33 +5235,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_nop 8
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -6699,33 +5261,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 8
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 0, i32 0, i32 0)
@@ -6840,24 +5375,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_nop 3
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -6877,25 +5394,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_nop 2
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -6914,25 +5412,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_nop 2
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 0, i32 0, i32 0)
@@ -7035,25 +5514,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -7067,25 +5527,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -7098,25 +5539,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <4 x float> %mai.1, ptr addrspace(1) %arg
@@ -7306,46 +5728,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 8
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 1.0
@@ -7362,46 +5744,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 7
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1.0
@@ -7417,46 +5759,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 7
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -7668,51 +5970,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, 0x40004000
-; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s2
-; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -7731,50 +5988,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
-; GFX942-GISEL-NEXT: s_mov_b32 s2, 0x40004000
-; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s2
-; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 8
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -7792,50 +6005,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, 0x40004000
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s2
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 8
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>, <4 x half> <half 2.0, half 2.0, half 2.0, half 2.0>, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -8155,27 +6324,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, 0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v32, 1.0
@@ -8196,26 +6344,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 1.0
@@ -8235,26 +6363,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
store <32 x float> %mai.1, ptr addrspace(1) %arg
@@ -8343,25 +6451,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, 2.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -8377,25 +6466,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_mov_b32 s1, 2.0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -8410,25 +6480,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 2.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <4 x float> %mai.1, ptr addrspace(1) %arg
@@ -8620,46 +6671,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s23, 2.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 8
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -8689,46 +6700,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s23, 2.0
-; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 7
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -8757,46 +6728,6 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 7
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 2.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -9153,75 +7084,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s37, 0
-; GFX90A-GISEL-NEXT: s_mov_b32 s36, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s38, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s39, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s40, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s41, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s42, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s43, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s44, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s45, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s46, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s47, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s48, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s49, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s50, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s51, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s52, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s53, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s54, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s55, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s56, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s57, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s58, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s59, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s60, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s61, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s62, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s63, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s64, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s65, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s66, s37
-; GFX90A-GISEL-NEXT: s_mov_b32 s67, s37
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 0
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -9289,74 +7151,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s37, 0
-; GFX942-GISEL-NEXT: s_mov_b32 s36, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s38, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s39, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s40, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s41, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s42, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s43, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s44, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s45, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s46, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s47, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s48, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s49, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s50, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s51, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s52, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s53, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s54, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s55, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s56, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s57, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s58, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s59, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s60, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s61, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s62, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s63, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s64, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s65, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s66, s37
-; GFX942-GISEL-NEXT: s_mov_b32 s67, s37
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -9423,74 +7217,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s37, 0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s36, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s38, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s39, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s40, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s41, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s42, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s43, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s44, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s45, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s46, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s47, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s48, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s49, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s50, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s51, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s52, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s53, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s54, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s55, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s56, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s57, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s58, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s59, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s60, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s61, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s62, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s63, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s64, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s65, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s66, s37
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s67, s37
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_nop 0
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 1.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
store <32 x float> %mai.1, ptr addrspace(1) %arg
@@ -9589,26 +7315,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v6, v[0:3]
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 3
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v5, 1.0
@@ -9627,26 +7333,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3]
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 2
-; GFX942-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0
@@ -9664,26 +7350,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-VGPR-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-VGPR-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3]
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 2
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
@@ -9784,25 +7450,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: s_nop 1
-; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v5, 1.0
@@ -9820,25 +7467,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0
@@ -9855,25 +7483,6 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
@@ -10199,36 +7808,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX90A-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GFX90A-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX90A-GISEL-NEXT: s_nop 15
-; GFX90A-GISEL-NEXT: s_nop 2
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX90A-GISEL-NEXT: s_endpgm
-;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -10259,36 +7838,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-NEXT: s_endpgm
;
-; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GFX942-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-GISEL-NEXT: s_nop 15
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-GISEL-NEXT: s_endpgm
-;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -10318,36 +7867,6 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: s_endpgm
-;
-; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
-; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
-; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-VGPR-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-VGPR-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
-; GFX942-VGPR-GISEL-NEXT: s_nop 15
-; GFX942-VGPR-GISEL-NEXT: s_nop 1
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index aeeb5958e9ca3..fe750dd483655 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -1,8 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -global-isel=0 < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GISEL %s
-; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -global-isel=0 --amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefixes=AGPR,AGPR-SDAG %s
-; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -global-isel=1 --amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefixes=AGPR,AGPR-GISEL %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,SDAG %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GISEL %s
; 0 = fp8
; 1 = bf8
@@ -21,22 +19,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -50,22 +32,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_1_1__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[0,0,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_1_1__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel:[1,1,0] op_sel_hi:[0,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -79,22 +45,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_2_2__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[1,1,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_2_2__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[1,1,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -108,22 +58,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_3_3__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,1,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_3_3__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,1,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -137,22 +71,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_3__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[0,1,0] op_sel_hi:[0,1,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_3__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel:[0,1,0] op_sel_hi:[0,1,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -166,22 +84,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_3_0__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,0,0] op_sel_hi:[1,0,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_3_0__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel:[1,0,0] op_sel_hi:[1,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -195,22 +97,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_2_3__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[0,1,0] op_sel_hi:[1,1,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_2_3__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -224,22 +110,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_3_2__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_3_2__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -254,22 +124,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 0, ; blgp
@@ -284,22 +138,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 1, ; blgp
@@ -314,22 +152,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp1__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19] blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp1__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 1, ; blgp
@@ -344,22 +166,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp2(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp2:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] blgp:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 2, ; blgp
@@ -374,22 +180,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp2__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17] blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp2__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] blgp:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 2, ; blgp
@@ -404,22 +194,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp3(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp3:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] blgp:3
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 3, ; blgp
@@ -434,22 +208,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp3__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17] blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp3__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] blgp:3
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 3, ; blgp
@@ -464,22 +222,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp4(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] blgp:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 4, ; blgp
@@ -494,22 +236,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp4__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:11], v[12:15] blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp4__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3] blgp:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 4, ; blgp
@@ -524,22 +250,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp0(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] cbsz:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 0, ; blgp
@@ -554,22 +264,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp0__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19] cbsz:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp0__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 0, ; blgp
@@ -584,22 +278,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp1(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] cbsz:1 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:1 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 1, ; blgp
@@ -615,22 +293,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp1__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19] cbsz:1 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp1__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:1 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 1, ; blgp
@@ -645,22 +307,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp2(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:1 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp2:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:1 blgp:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 2, ; blgp
@@ -674,22 +320,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp2__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17] cbsz:1 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp2__constant_scale_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] cbsz:1 blgp:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 2, ; blgp
@@ -704,22 +334,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp3(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:1 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp3:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:1 blgp:3
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 3, ; blgp
@@ -734,22 +348,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp3__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17] cbsz:1 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp3__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3] cbsz:1 blgp:3
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 3, ; blgp
@@ -764,22 +362,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp4(<8 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:1 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:1 blgp:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 4, ; blgp
@@ -794,22 +376,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp4__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:11], v[12:15] cbsz:1 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz1__blgp4__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:11], a[0:3] cbsz:1 blgp:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 1, ; cbsz
i32 4, ; blgp
@@ -824,22 +390,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp0(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 0, ; blgp
@@ -854,22 +404,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp0__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17] cbsz:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp0__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 0, ; blgp
@@ -884,22 +418,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp1(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:2 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:2 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 1, ; blgp
@@ -914,22 +432,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp1__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17] cbsz:2 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp1__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:2 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 1, ; blgp
@@ -944,22 +446,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:2 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:2 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 2, ; blgp
@@ -974,22 +460,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15] cbsz:2 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:2 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 2, ; blgp
@@ -1004,22 +474,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp3(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:2 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp3:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:2 blgp:3
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 3, ; blgp
@@ -1034,22 +488,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp3__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15] cbsz:2 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp3__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:2 blgp:3
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 3, ; blgp
@@ -1065,22 +503,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp0(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:3
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 0, ; blgp
@@ -1095,22 +517,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp0__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17] cbsz:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp0__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:3
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 0, ; blgp
@@ -1125,22 +531,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp1(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:3 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:3 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 1, ; blgp
@@ -1155,22 +545,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp1__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17] cbsz:3 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp1__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3] cbsz:3 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 1, ; blgp
@@ -1185,22 +559,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp2(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:3 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp2:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:3 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 2, ; blgp
@@ -1215,22 +573,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp2__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15] cbsz:3 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp2__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:3 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 2, ; blgp
@@ -1245,22 +587,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp4(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:9], v[10:13], v14, v15 op_sel_hi:[0,0,0] cbsz:3 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:3 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v4i32(<6 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 4, ; blgp
@@ -1275,22 +601,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp4__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:9], v[10:13] cbsz:3 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp4__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3] cbsz:3 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v4i32(<6 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 4, ; blgp
@@ -1305,22 +615,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp3(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:3 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp3:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:3 blgp:3
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 3, ; blgp
@@ -1335,22 +629,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp3__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:11], v[12:15] cbsz:3 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz3__blgp3__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:3 blgp:3
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 3, ; cbsz
i32 3, ; blgp
@@ -1365,22 +643,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp4(<6 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:9], v[10:13], v14, v15 op_sel_hi:[0,0,0] cbsz:2 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:2 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v4i32(<6 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 4, ; blgp
@@ -1395,22 +657,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp4__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:9], v[10:13] cbsz:2 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp4__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:9], a[0:3] cbsz:2 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v4i32(<6 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 4, ; blgp
@@ -1425,22 +671,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp0(<4 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v8i32(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 0, ; blgp
@@ -1455,22 +685,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp0__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:11], v[12:15] cbsz:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp0__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3] cbsz:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v8i32(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 0, ; blgp
@@ -1485,22 +699,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp1(<4 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:11], v[12:15], v16, v17 op_sel_hi:[0,0,0] cbsz:4 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp1:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3], v16, v17 op_sel_hi:[0,0,0] cbsz:4 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v8i32(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 1, ; blgp
@@ -1515,22 +713,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp1__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:11], v[12:15] cbsz:4 blgp:1
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp1__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v13
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v15
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:11], a[0:3] cbsz:4 blgp:1
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v8i32(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 1, ; blgp
@@ -1545,22 +727,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp2(<4 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:9], v[10:13], v14, v15 op_sel_hi:[0,0,0] cbsz:4 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp2:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:4 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v6i32(<4 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 2, ; blgp
@@ -1575,22 +741,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp2__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:9], v[10:13] cbsz:4 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp2__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3] cbsz:4 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v6i32(<4 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 2, ; blgp
@@ -1605,22 +755,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp3(<4 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:9], v[10:13], v14, v15 op_sel_hi:[0,0,0] cbsz:4 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp3:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3], v14, v15 op_sel_hi:[0,0,0] cbsz:4 blgp:3
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v6i32(<4 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 3, ; blgp
@@ -1635,22 +769,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp3__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:9], v[10:13] cbsz:4 blgp:3
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp3__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v11
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v12
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v13
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:9], a[0:3] cbsz:4 blgp:3
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v6i32(<4 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 3, ; blgp
@@ -1665,22 +783,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4(<4 x
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:7], v[8:11], v12, v13 op_sel_hi:[0,0,0] cbsz:4 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:7], a[0:3], v12, v13 op_sel_hi:[0,0,0] cbsz:4 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v4i32(<4 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 4, ; blgp
@@ -1695,22 +797,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4__cons
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:4 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4__constant_scale_0_0:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:4 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v4i32(<4 x i32> %arg0, <4 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 4, ; blgp
@@ -1731,24 +817,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__sgpr_scaleA__sgpr_
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__sgpr_scaleA__sgpr_scaleB:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: v_mov_b32_e32 v16, s0
-; AGPR-NEXT: v_mov_b32_e32 v17, s1
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[0,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -1761,23 +829,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__sgpr_scaleA__vgpr_
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v20 op_sel_hi:[0,0,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__sgpr_scaleA__vgpr_scaleB:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: v_mov_b32_e32 v16, s0
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v20 op_sel_hi:[0,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -1790,23 +841,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__vgpr_scaleA__sgpr_
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__vgpr_scaleA__sgpr_scaleB:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: v_mov_b32_e32 v16, s0
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v16 op_sel_hi:[0,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -1874,70 +908,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs(<8 x i32> inr
; GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s1
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s3
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_readfirstlane_b32 s0, v1
-; AGPR-SDAG-NEXT: v_readfirstlane_b32 s1, v0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s23
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s27
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, s28
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, s29
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, s1
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, s0
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[12:19], v[4:11], a[0:3], v2, v3 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgprs:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: s_mov_b32 s12, s0
-; AGPR-GISEL-NEXT: s_mov_b32 s13, s1
-; AGPR-GISEL-NEXT: s_mov_b32 s14, s2
-; AGPR-GISEL-NEXT: s_mov_b32 s15, s3
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; AGPR-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; AGPR-GISEL-NEXT: v_readfirstlane_b32 s1, v1
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, s28
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, s29
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, s0
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, s1
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[4:11], v[12:19], a[0:3], v2, v3 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -1974,56 +944,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__sgp
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[14:21], v[0:7], v[8:11], v13, v12 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__sgpr_vgpr:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s1
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s3
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s19
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], v8, v12 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__sgpr_vgpr:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: s_mov_b32 s12, s0
-; AGPR-GISEL-NEXT: s_mov_b32 s13, s1
-; AGPR-GISEL-NEXT: s_mov_b32 s14, s2
-; AGPR-GISEL-NEXT: s_mov_b32 s15, s3
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], v8, v12 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -2060,56 +980,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__vgp
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[14:21], v[0:7], v[8:11], v12, v13 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__vgpr_sgpr:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s1
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s3
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s19
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], v12, v8 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_vgpr__vgpr_sgpr:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: s_mov_b32 s12, s0
-; AGPR-GISEL-NEXT: s_mov_b32 s13, s1
-; AGPR-GISEL-NEXT: s_mov_b32 s14, s2
-; AGPR-GISEL-NEXT: s_mov_b32 s15, s3
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[14:21], v[0:7], a[0:3], v12, v8 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -2146,56 +1016,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_sgpr_vgpr__vgp
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[14:21], v[8:11], v12, v13 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_sgpr_vgpr__vgpr_sgpr:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s1
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s3
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s19
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[14:21], a[0:3], v12, v8 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_sgpr_vgpr__vgpr_sgpr:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: s_mov_b32 s12, s0
-; AGPR-GISEL-NEXT: s_mov_b32 s13, s1
-; AGPR-GISEL-NEXT: s_mov_b32 s14, s2
-; AGPR-GISEL-NEXT: s_mov_b32 s15, s3
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v8
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v9
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v10
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v11
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v8, s20
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[14:21], a[0:3], v12, v8 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -2222,23 +1042,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_vgpr_sgpr__vgp
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[18:21], v16, v17 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_vgpr_vgpr_sgpr__vgpr_sgpr:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
-; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
-; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
-; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
-; AGPR-NEXT: v_mov_b32_e32 v17, s16
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[0,0,0]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -2281,56 +1084,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_sgpr__vgp
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[10:17], v[0:7], v[18:21], v8, v9 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_sgpr__vgpr_sgpr:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s1
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s3
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s19
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, s20
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, s21
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, s22
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, s23
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s24
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[10:17], v[0:7], a[0:3], v8, v9 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0_sgpr_vgpr_sgpr__vgpr_sgpr:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: s_mov_b32 s12, s0
-; AGPR-GISEL-NEXT: s_mov_b32 s13, s1
-; AGPR-GISEL-NEXT: s_mov_b32 s14, s2
-; AGPR-GISEL-NEXT: s_mov_b32 s15, s3
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, s20
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, s21
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, s22
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, s23
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v9, s24
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[10:17], v[0:7], a[0:3], v8, v9 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 %scale0, i32 0, i32 %scale1)
ret <4 x float> %result
}
@@ -2353,42 +1106,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_inlineimm__
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[1,1,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_inlineimm__scaleB_inlineimm:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 33
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v21, v20 op_sel_hi:[1,1,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_inlineimm__scaleB_inlineimm:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v16, 33
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v17, -2
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[1,1,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 2, i32 33, i32 2, i32 -2)
ret <4 x float> %result
}
@@ -2411,42 +1128,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scale
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[1,1,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scaleB_inlineimm:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 0x41
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v21, v20 op_sel_hi:[1,1,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scaleB_inlineimm:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0x41
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v17, -2
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[1,1,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 2, i32 65, i32 2, i32 -2)
ret <4 x float> %result
}
@@ -2469,42 +1150,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scale
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[1,1,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scaleB_kimm:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0x4d
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 0x41
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v21, v20 op_sel_hi:[1,1,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4_0_0__scaleA_kimm__scaleB_kimm:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0x41
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v17, 0x4d
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[1,1,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 2, i32 65, i32 2, i32 77)
ret <4 x float> %result
}
@@ -2513,35 +1158,35 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
+; SDAG-NEXT: v_mov_b32_e32 v6, s8
+; SDAG-NEXT: v_mov_b32_e32 v7, s9
+; SDAG-NEXT: v_mov_b32_e32 v8, s10
+; SDAG-NEXT: v_mov_b32_e32 v9, s11
+; SDAG-NEXT: v_mov_b32_e32 v10, s12
+; SDAG-NEXT: v_mov_b32_e32 v11, s13
+; SDAG-NEXT: v_mov_b32_e32 v12, s14
+; SDAG-NEXT: v_mov_b32_e32 v13, s15
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v14, s16
+; SDAG-NEXT: v_mov_b32_e32 v15, s17
+; SDAG-NEXT: v_mov_b32_e32 v16, s18
+; SDAG-NEXT: v_mov_b32_e32 v17, s19
+; SDAG-NEXT: v_mov_b32_e32 v18, s20
+; SDAG-NEXT: v_mov_b32_e32 v19, s21
+; SDAG-NEXT: v_mov_b32_e32 v20, s22
+; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v5, s12
-; SDAG-NEXT: v_mov_b32_e32 v6, s13
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
-; SDAG-NEXT: s_nop 11
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[6:13], v[14:21], v[0:3], v4, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
; SDAG-NEXT: s_endpgm
;
@@ -2568,65 +1213,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; GISEL-NEXT: s_nop 10
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[30:31]
; GISEL-NEXT: s_endpgm
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s13
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
-; AGPR-SDAG-NEXT: s_endpgm
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x40
-; AGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v20, s28
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v21, s29
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-GISEL-NEXT: s_nop 10
-; AGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[30:31]
-; AGPR-GISEL-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 2, i32 3, i32 %scale0, i32 1, i32 %scale1)
store <4 x float> %result, ptr addrspace(1) %ptr, align 16
ret void
@@ -2637,33 +1223,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, -2
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v4, -2
+; SDAG-NEXT: v_mov_b32_e32 v5, 0x41
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v6, s8
+; SDAG-NEXT: v_mov_b32_e32 v7, s9
+; SDAG-NEXT: v_mov_b32_e32 v8, s10
+; SDAG-NEXT: v_mov_b32_e32 v9, s11
+; SDAG-NEXT: v_mov_b32_e32 v10, s12
+; SDAG-NEXT: v_mov_b32_e32 v11, s13
+; SDAG-NEXT: v_mov_b32_e32 v12, s14
+; SDAG-NEXT: v_mov_b32_e32 v13, s15
+; SDAG-NEXT: v_mov_b32_e32 v14, s16
+; SDAG-NEXT: v_mov_b32_e32 v15, s17
+; SDAG-NEXT: v_mov_b32_e32 v16, s18
+; SDAG-NEXT: v_mov_b32_e32 v17, s19
+; SDAG-NEXT: v_mov_b32_e32 v18, s20
+; SDAG-NEXT: v_mov_b32_e32 v19, s21
+; SDAG-NEXT: v_mov_b32_e32 v20, s22
+; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[6:13], v[14:21], v[0:3], v5, v4 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
@@ -2690,64 +1276,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; GISEL-NEXT: s_nop 10
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-SDAG-NEXT: s_endpgm
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v20, 0x41
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v21, -2
-; AGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-GISEL-NEXT: s_nop 10
-; AGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-GISEL-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 65, i32 1, i32 -2)
store <4 x float> %result, ptr addrspace(1) %ptr, align 16
ret void
@@ -2758,33 +1286,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v4, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v5, 0x41
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v6, s8
+; SDAG-NEXT: v_mov_b32_e32 v7, s9
+; SDAG-NEXT: v_mov_b32_e32 v8, s10
+; SDAG-NEXT: v_mov_b32_e32 v9, s11
+; SDAG-NEXT: v_mov_b32_e32 v10, s12
+; SDAG-NEXT: v_mov_b32_e32 v11, s13
+; SDAG-NEXT: v_mov_b32_e32 v12, s14
+; SDAG-NEXT: v_mov_b32_e32 v13, s15
+; SDAG-NEXT: v_mov_b32_e32 v14, s16
+; SDAG-NEXT: v_mov_b32_e32 v15, s17
+; SDAG-NEXT: v_mov_b32_e32 v16, s18
+; SDAG-NEXT: v_mov_b32_e32 v17, s19
+; SDAG-NEXT: v_mov_b32_e32 v18, s20
+; SDAG-NEXT: v_mov_b32_e32 v19, s21
+; SDAG-NEXT: v_mov_b32_e32 v20, s22
+; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[6:13], v[14:21], v[0:3], v5, v4 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
@@ -2811,64 +1339,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; GISEL-NEXT: s_nop 10
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-SDAG-NEXT: s_endpgm
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v20, 0x41
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v21, 1.0
-; AGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-GISEL-NEXT: s_nop 10
-; AGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-GISEL-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 65, i32 1, i32 1065353216)
store <4 x float> %result, ptr addrspace(1) %ptr, align 16
ret void
@@ -2879,33 +1349,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, -2
-; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v4, -2
+; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v6, s8
+; SDAG-NEXT: v_mov_b32_e32 v7, s9
+; SDAG-NEXT: v_mov_b32_e32 v8, s10
+; SDAG-NEXT: v_mov_b32_e32 v9, s11
+; SDAG-NEXT: v_mov_b32_e32 v10, s12
+; SDAG-NEXT: v_mov_b32_e32 v11, s13
+; SDAG-NEXT: v_mov_b32_e32 v12, s14
+; SDAG-NEXT: v_mov_b32_e32 v13, s15
+; SDAG-NEXT: v_mov_b32_e32 v14, s16
+; SDAG-NEXT: v_mov_b32_e32 v15, s17
+; SDAG-NEXT: v_mov_b32_e32 v16, s18
+; SDAG-NEXT: v_mov_b32_e32 v17, s19
+; SDAG-NEXT: v_mov_b32_e32 v18, s20
+; SDAG-NEXT: v_mov_b32_e32 v19, s21
+; SDAG-NEXT: v_mov_b32_e32 v20, s22
+; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[6:13], v[14:21], v[0:3], v5, v4 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
@@ -2932,64 +1402,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; GISEL-NEXT: s_nop 10
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-SDAG-NEXT: s_endpgm
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v20, 1.0
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v21, -2
-; AGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-GISEL-NEXT: s_nop 10
-; AGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-GISEL-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 1065353216, i32 1, i32 -2)
store <4 x float> %result, ptr addrspace(1) %ptr, align 16
ret void
@@ -3000,33 +1412,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
-; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v4, 0.15915494
+; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
+; SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x50
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v6, s8
+; SDAG-NEXT: v_mov_b32_e32 v7, s9
+; SDAG-NEXT: v_mov_b32_e32 v8, s10
+; SDAG-NEXT: v_mov_b32_e32 v9, s11
+; SDAG-NEXT: v_mov_b32_e32 v10, s12
+; SDAG-NEXT: v_mov_b32_e32 v11, s13
+; SDAG-NEXT: v_mov_b32_e32 v12, s14
+; SDAG-NEXT: v_mov_b32_e32 v13, s15
+; SDAG-NEXT: v_mov_b32_e32 v14, s16
+; SDAG-NEXT: v_mov_b32_e32 v15, s17
+; SDAG-NEXT: v_mov_b32_e32 v16, s18
+; SDAG-NEXT: v_mov_b32_e32 v17, s19
+; SDAG-NEXT: v_mov_b32_e32 v18, s20
+; SDAG-NEXT: v_mov_b32_e32 v19, s21
+; SDAG-NEXT: v_mov_b32_e32 v20, s22
+; SDAG-NEXT: v_mov_b32_e32 v21, s23
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[6:13], v[14:21], v[0:3], v5, v4 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_nop 10
+; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
@@ -3053,64 +1465,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; GISEL-NEXT: s_nop 10
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-SDAG-NEXT: s_endpgm
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
-; AGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v20, 1.0
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v21, 0.15915494
-; AGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-GISEL-NEXT: s_nop 10
-; AGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
-; AGPR-GISEL-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 1065353216, i32 1, i32 1042479491)
store <4 x float> %result, ptr addrspace(1) %ptr, align 16
ret void
@@ -3123,22 +1477,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_a(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_a:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
ret <4 x float> %result
}
@@ -3150,22 +1488,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_b(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19]
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_0_b:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3]
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 3, i32 0, i32 1, i32 0)
ret <4 x float> %result
}
@@ -3188,42 +1510,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_1(<8
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_1:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 1
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 0
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v21, v20 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_0_1:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v17, 1
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1)
ret <4 x float> %result
}
@@ -3246,42 +1532,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_1_0_a(
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0]
; GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_1_0_a:
-; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 1
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v21, v20 op_sel_hi:[0,0,0]
-; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___constant_scale_1_0_a:
-; AGPR-GISEL: ; %bb.0:
-; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-GISEL-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
-; AGPR-GISEL-NEXT: v_mov_b32_e32 v17, 0
-; AGPR-GISEL-NEXT: s_nop 1
-; AGPR-GISEL-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[0,0,0]
-; AGPR-GISEL-NEXT: s_nop 11
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0)
ret <4 x float> %result
}
@@ -3296,22 +1546,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v8i32_fp6(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v8i32_fp6:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] blgp:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 2, ; blgp
@@ -3325,22 +1559,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp8(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] cbsz:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp8:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:2
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 0, ; blgp
@@ -3354,22 +1572,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp6(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] cbsz:2 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp6:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:2 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 2, ; blgp
@@ -3383,22 +1585,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp6_
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19] cbsz:2 blgp:2
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp6__v8i32_fp6__0_scale:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:2 blgp:2
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 2, ; cbsz
i32 2, ; blgp
@@ -3412,22 +1598,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v8i32_fp4(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v8i32_fp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] blgp:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 4, ; blgp
@@ -3441,22 +1611,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp8(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] cbsz:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp8:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 0, ; blgp
@@ -3470,22 +1624,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v6i32_fp4(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp8__v6i32_fp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] blgp:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %arg0, <6 x i32> %arg1, <4 x float> %arg2,
i32 0, ; cbsz
i32 4, ; blgp
@@ -3499,22 +1637,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v6i32_fp4__v8i32_fp8(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:5], v[6:13], v[14:17], v18, v19 op_sel_hi:[0,0,0] cbsz:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v6i32_fp4__v8i32_fp8:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v14
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v15
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v17
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:13], a[0:3], v18, v19 op_sel_hi:[0,0,0] cbsz:4
-; AGPR-NEXT: s_nop 11
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 0, ; blgp
@@ -3528,22 +1650,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp4(
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v20, v21 op_sel_hi:[0,0,0] cbsz:4 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp4:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v20, v21 op_sel_hi:[0,0,0] cbsz:4 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 4, ; blgp
@@ -3557,22 +1663,6 @@ define <4 x float> @test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp4_
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19] cbsz:4 blgp:4
; GCN-NEXT: s_setpc_b64 s[30:31]
-;
-; AGPR-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4___v8i32_fp4__v8i32_fp4__0_scale:
-; AGPR: ; %bb.0:
-; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; AGPR-NEXT: v_accvgpr_write_b32 a0, v16
-; AGPR-NEXT: v_accvgpr_write_b32 a1, v17
-; AGPR-NEXT: v_accvgpr_write_b32 a2, v18
-; AGPR-NEXT: v_accvgpr_write_b32 a3, v19
-; AGPR-NEXT: s_nop 1
-; AGPR-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] cbsz:4 blgp:4
-; AGPR-NEXT: s_nop 7
-; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
-; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
-; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
-; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
-; AGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2,
i32 4, ; cbsz
i32 4, ; blgp
diff --git a/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir b/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir
index 702257251228a..dfc2e0bc93d7d 100644
--- a/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir
+++ b/llvm/test/CodeGen/AMDGPU/neighboring-mfma-padding.mir
@@ -221,22 +221,21 @@ body: |
;
; gfx908-PAD25-LABEL: name: mfma_padding_8_pass
; gfx908-PAD25: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD25-NEXT: S_NOP 1
; gfx908-PAD25-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD50-LABEL: name: mfma_padding_8_pass
; gfx908-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD50-NEXT: S_NOP 3
+ ; gfx908-PAD50-NEXT: S_NOP 0
; gfx908-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD75-LABEL: name: mfma_padding_8_pass
; gfx908-PAD75: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD75-NEXT: S_NOP 5
+ ; gfx908-PAD75-NEXT: S_NOP 0
; gfx908-PAD75-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD100-LABEL: name: mfma_padding_8_pass
; gfx908-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx908-PAD100-NEXT: S_NOP 7
+ ; gfx908-PAD100-NEXT: S_NOP 1
; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-DEFAULT-LABEL: name: mfma_padding_8_pass
@@ -245,12 +244,12 @@ body: |
;
; gfx90a-PAD50-LABEL: name: mfma_padding_8_pass
; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx90a-PAD50-NEXT: S_NOP 3
+ ; gfx90a-PAD50-NEXT: S_NOP 0
; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-PAD100-LABEL: name: mfma_padding_8_pass
; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx90a-PAD100-NEXT: S_NOP 7
+ ; gfx90a-PAD100-NEXT: S_NOP 1
; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-DEFAULT-LABEL: name: mfma_padding_8_pass
@@ -259,12 +258,12 @@ body: |
;
; gfx942-PAD50-LABEL: name: mfma_padding_8_pass
; gfx942-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx942-PAD50-NEXT: S_NOP 3
+ ; gfx942-PAD50-NEXT: S_NOP 0
; gfx942-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-PAD100-LABEL: name: mfma_padding_8_pass
; gfx942-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
- ; gfx942-PAD100-NEXT: S_NOP 7
+ ; gfx942-PAD100-NEXT: S_NOP 1
; gfx942-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
@@ -290,21 +289,18 @@ body: |
; gfx908-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx908-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx908-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx908-PAD50-NEXT: S_NOP 1
; gfx908-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD75-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx908-PAD75: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx908-PAD75-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx908-PAD75-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx908-PAD75-NEXT: S_NOP 3
; gfx908-PAD75-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx908-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx908-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx908-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx908-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx908-PAD100-NEXT: S_NOP 5
; gfx908-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-DEFAULT-LABEL: name: mfma_padding_8_pass_2_intervening_valu
@@ -317,14 +313,12 @@ body: |
; gfx90a-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx90a-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx90a-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx90a-PAD50-NEXT: S_NOP 1
; gfx90a-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx90a-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx90a-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx90a-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx90a-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx90a-PAD100-NEXT: S_NOP 5
; gfx90a-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-DEFAULT-LABEL: name: mfma_padding_8_pass_2_intervening_valu
@@ -337,14 +331,12 @@ body: |
; gfx942-PAD50: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx942-PAD50-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx942-PAD50-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx942-PAD50-NEXT: S_NOP 1
; gfx942-PAD50-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
;
; gfx942-PAD100-LABEL: name: mfma_padding_8_pass_2_intervening_valu
; gfx942-PAD100: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
; gfx942-PAD100-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
; gfx942-PAD100-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
- ; gfx942-PAD100-NEXT: S_NOP 5
; gfx942-PAD100-NEXT: early-clobber $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
$vgpr2 = V_MOV_B32_e32 1, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index dec44a092485f..afb71ee60494a 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgpu9.42--amdhsa %s -o - | FileCheck %s --check-prefixes=GFX942
-; RUN: llc -mtriple=amdgpu9.08--amdhsa %s -o - | FileCheck %s --check-prefixes=GFX908
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx942 %s -o - | FileCheck %s --check-prefixes=GFX942
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx908 %s -o - | FileCheck %s --check-prefixes=GFX908
define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX942-LABEL: matmul_kernel:
@@ -52,43 +52,42 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
; GFX908-NEXT: s_mov_b32 s0, 0
-; GFX908-NEXT: s_mov_b32 s1, 0
+; GFX908-NEXT: s_mov_b32 s6, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_cmp_lg_u32 s2, 0
; GFX908-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX908-NEXT: s_branch .LBB0_3
; GFX908-NEXT: .LBB0_1: ; %bb2
; GFX908-NEXT: ; in Loop: Header=BB0_3 Depth=1
-; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_accvgpr_read_b32 v3, a2
-; GFX908-NEXT: s_or_b32 s4, s1, 1
-; GFX908-NEXT: s_ashr_i32 s5, s1, 31
; GFX908-NEXT: s_mov_b32 s1, s0
; GFX908-NEXT: v_mov_b32_e32 v2, s1
; GFX908-NEXT: v_mov_b32_e32 v1, s0
; GFX908-NEXT: v_accvgpr_write_b32 a0, v3
; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
; GFX908-NEXT: v_accvgpr_read_b32 v3, a1
-; GFX908-NEXT: s_and_b32 s1, s5, s4
+; GFX908-NEXT: s_or_b32 s1, s6, 1
; GFX908-NEXT: v_accvgpr_write_b32 a2, v4
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: s_ashr_i32 s4, s6, 31
; GFX908-NEXT: v_mfma_f32_16x16x16f16 a[2:5], v[1:2], v[1:2], a[0:3]
+; GFX908-NEXT: s_and_b32 s6, s4, s1
+; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB0_2: ; %Flow
; GFX908-NEXT: ; in Loop: Header=BB0_3 Depth=1
; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX908-NEXT: s_cselect_b32 s4, 1, 0
-; GFX908-NEXT: s_cmp_lg_u32 s4, 1
+; GFX908-NEXT: s_cselect_b32 s1, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s1, 1
; GFX908-NEXT: s_cbranch_scc0 .LBB0_5
; GFX908-NEXT: .LBB0_3: ; %bb
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_and_b64 s[4:5], s[2:3], exec
-; GFX908-NEXT: s_cselect_b32 s4, 1, 0
-; GFX908-NEXT: s_cmp_lg_u32 s4, 1
+; GFX908-NEXT: s_cselect_b32 s1, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s1, 1
; GFX908-NEXT: s_cbranch_scc0 .LBB0_1
; GFX908-NEXT: ; %bb.4: ; in Loop: Header=BB0_3 Depth=1
; GFX908-NEXT: s_mov_b64 s[4:5], -1
-; GFX908-NEXT: ; implicit-def: $sgpr1
+; GFX908-NEXT: ; implicit-def: $sgpr6
; GFX908-NEXT: ; implicit-def: $agpr2
; GFX908-NEXT: s_branch .LBB0_2
; GFX908-NEXT: .LBB0_5: ; %common.ret
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir b/llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir
new file mode 100644
index 0000000000000..8eca07927bea7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/schedule-mfma-valu-after-load.mir
@@ -0,0 +1,31 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx950 -run-pass=machine-scheduler --misched-prera-direction=topdown -verify-machineinstrs %s -o - -debug-only=machine-scheduler 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# BUFFER_LOAD (multi-cycle) -> 8-pass MFMA -> VALU consuming load data.
+# VALU stalls for operand readiness; no spurious HWVALU hazard from MFMA.
+
+# CHECK: ScheduleDAGMILive::schedule starting
+# CHECK: Scheduling SU({{[0-9]+}}) {{.*}} BUFFER_LOAD_DWORDX4
+# CHECK: Scheduling SU({{[0-9]+}}) {{.*}} V_MFMA_F32_32X32X16_F16
+# CHECK: HWVALU +2x1u
+# CHECK: HWXDL +8x1u
+# CHECK: Scheduling SU({{[0-9]+}}) {{.*}} V_ADD_F32
+# CHECK: *** Stall until: [[STALL:[0-9]+]]
+# CHECK-NOT: hazard: SU({{[0-9]+}}) HWVALU[0]={{[0-9]+}}c, is later than CurrCycle = {{[0-9]+}}c
+
+---
+name: mfma_valu_after_load
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+ %0:sgpr_128 = IMPLICIT_DEF
+ %1:vreg_128_align2 = IMPLICIT_DEF
+ %2:vreg_128_align2 = IMPLICIT_DEF
+ %3:vreg_512_align2 = IMPLICIT_DEF
+ %14:vgpr_32 = IMPLICIT_DEF
+ %4:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET %0, 0, 0, 0, 0, implicit $exec
+ %3:vreg_512_align2 = V_MFMA_F32_32X32X16_F16_mac_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+ %5:vgpr_32 = nofpexcept V_ADD_F32_e32 %4.sub0, %14, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %3, implicit %5
+...
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir b/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir
index c5cf47fc06065..eece4608e5e83 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-pending-queue.mir
@@ -3,12 +3,15 @@
# Check that cycle counts are consistent with hazards.
-# CHECK: Cycle: 3 TopQ.A
-# CHECK: hazard: SU(6) HWXDL[0]=9c, is later than CurrCycle = 3c
-# CHECK-NOT: Cycle: 9 TopQ.A
-# CHECK: Cycle: 83 TopQ.A
# CHECK: Checking pending node SU(6)
-# CHECK: Move SU(6) into Available Q
+# CHECK: hazard: SU(6) HWVALU[0]=3c, is later than CurrCycle = 2c
+# CHECK: Scheduling SU(7) %5.sub0:vreg_128_align2 = V_ADD_U32_e32 %4.sub0:vreg_128, %1:vgpr_32, implicit $exec
+# CHECK: *** Stall until: 80
+# CHECK: HWVALU +1x1u
+# CHECK: Resource conflict: HWVALU[0] reserved until @3
+# CHECK: Scheduling SU(6) %3:areg_512 = V_MFMA_F32_16X16X1F32_mac_e64 %2:vgpr_32, %1:vgpr_32, %3:areg_512(tied-def 0), 0, 0, 0, implicit $mode, implicit $exec
+# CHECK: HWVALU +2x1u
+# CHECK: HWXDL +8x1u
---
name: pending_queue_ready_cycle
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
index 5685df559b082..e6f1316ecf231 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
@@ -31,8 +31,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_pk_mul_unpacking_f32
@@ -53,8 +52,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_pk_mul_unpacking_f32
@@ -124,8 +122,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_op_sel_selection_unpacking_f32
@@ -146,8 +143,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_op_sel_selection_unpacking_f32
@@ -217,8 +213,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_op_sel_hi_selection_unpacking_f32
@@ -239,8 +234,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_op_sel_hi_selection_unpacking_f32
@@ -322,12 +316,9 @@ body: |
; GFX950-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr18, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr19, implicit $exec, implicit $exec
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_BF8_BF8_e64 killed $vgpr4_vgpr5, killed $vgpr2_vgpr3, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 1, 2, 3, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr2_vgpr3 = nofpexcept V_PK_ADD_F32 8, killed $sgpr2_sgpr3, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr6_vgpr7 = nofpexcept V_PK_ADD_F32 8, killed $sgpr6_sgpr7, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_ADD_F32 8, killed $sgpr4_sgpr5, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
@@ -361,12 +352,9 @@ body: |
; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr18, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr19, implicit $exec, implicit $exec
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_BF8_BF8_e64 killed $vgpr4_vgpr5, killed $vgpr2_vgpr3, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 1, 2, 3, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr2 = nofpexcept V_ADD_F32_e64 0, killed $sgpr2, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr3 = nofpexcept V_ADD_F32_e64 0, killed $sgpr3, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr6 = nofpexcept V_ADD_F32_e64 0, killed $sgpr6, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr7 = nofpexcept V_ADD_F32_e64 0, killed $sgpr7, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr4 = nofpexcept V_ADD_F32_e64 0, killed $sgpr4, 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr5 = nofpexcept V_ADD_F32_e64 0, killed $sgpr5, 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr2_vgpr3 = nofpexcept V_PK_ADD_F32 8, killed $sgpr2_sgpr3, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr6_vgpr7 = nofpexcept V_PK_ADD_F32 8, killed $sgpr6_sgpr7, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_ADD_F32 8, killed $sgpr4_sgpr5, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -465,8 +453,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 8, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_pk_fma_unpacking_f32
@@ -485,8 +472,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, killed $vgpr4, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 8, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_pk_fma_unpacking_f32
@@ -550,8 +536,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_MUL_F32 8, $sgpr30_sgpr31, 12, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_FMA_F32 8, killed $sgpr30_sgpr31, 8, $vgpr4_vgpr5, 0, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
@@ -571,8 +556,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr4 = nofpexcept V_MUL_F32_e64 0, $sgpr30, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr5 = nofpexcept V_MUL_F32_e64 0, $sgpr31, 0, $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_MUL_F32 8, $sgpr30_sgpr31, 12, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: renamable $vgpr4_vgpr5 = nofpexcept V_PK_FMA_F32 8, killed $sgpr30_sgpr31, 8, $vgpr4_vgpr5, 0, $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -740,8 +724,7 @@ body: |
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 $sgpr15, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr6 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr7 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 8, killed $vgpr6_vgpr7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_opsel_register_is_correctly_marked_as_killed
@@ -762,8 +745,7 @@ body: |
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 $sgpr15, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr6 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr7 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_FMA_F32_e64 0, $sgpr30, 0, $vgpr5, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_FMA_F32_e64 0, killed $sgpr30, 0, killed $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_FMA_F32 0, killed $sgpr30_sgpr31, 12, killed $vgpr4_vgpr5, 8, killed $vgpr6_vgpr7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_opsel_register_is_correctly_marked_as_killed
@@ -1011,8 +993,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, 1065353216, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_unpacking_with_imm_input
@@ -1033,8 +1014,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, 1065353216, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, 1065353216, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_unpacking_with_imm_input
@@ -1104,8 +1084,7 @@ body: |
; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX950-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX950-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 11, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: test_neg_lo_hi_post_unpacking
@@ -1126,8 +1105,7 @@ body: |
; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr16 = nofpexcept V_MUL_F32_e64 0, killed $sgpr30, 1, killed $vgpr4, 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: $vgpr17 = nofpexcept V_MUL_F32_e64 0, killed $sgpr31, 1, killed $vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = nofpexcept V_PK_MUL_F32 8, killed $sgpr30_sgpr31, 11, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
; GFX90A-LABEL: name: test_neg_lo_hi_post_unpacking
diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s
index 9a76a4bc7d640..95123eaf0d258 100644
--- a/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s
+++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx942-mfma.s
@@ -143,8 +143,8 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK: [6] - HWXDL
# CHECK: [0] [1] [2] [3] [4] [5] [6] Instructions:
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3]
@@ -153,54 +153,54 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - 8.00 - - v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 v[0:31], v0, v1, v[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 v[0:31], v0, v1, v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 a[0:31], v0, v1, a[34:65]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 v[10:13], a[2:3], v[4:7], v0 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 a[10:13], v[2:3], a[4:7], v1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 v[10:13], a[2:3], v[4:7], v4 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 a[10:13], v[2:3], a[4:7], v5
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 v[10:13], a[2:3], v[4:7], v8 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 a[10:13], v[2:3], a[4:7], v9
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
@@ -209,15 +209,15 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s
index bcdcc54855b4b..f9fa035286b20 100644
--- a/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s
+++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx950.s
@@ -189,63 +189,63 @@ v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK: [0] [1] [2] [3] [4] [5] [6] Instructions:
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[0:3], a[4:7]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_f16 v[0:15], v[0:3], v[0:3], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_f16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf16 v[0:15], v[0:3], v[0:3], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[0:3], a[4:7]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x32_i8 v[0:15], v[0:3], v[0:3], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x32_i8 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[0:3], a[4:7]
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[0:3], a[4:7]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_f16 v[0:15], v[0:3], v[0:3], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_f16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf16 v[0:15], v[0:3], v[0:3], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf16 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[0:3], a[4:7]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x32_i8 v[0:15], v[0:3], v[0:3], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x32_i8 a[0:15], a[0:3], a[0:3], a[0:15] blgp:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], a[0:3], a[0:3], a[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[0:3], a[4:7]
# CHECK-NEXT: - - - - 1.00 - - v_mfma_ld_scale_b32 v0, v0
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3]
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:3
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:7], v[0:3] blgp:4
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] cbsz:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:3
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:7], v[4:11], v[0:3] cbsz:4
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2 blgp:1
-# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] cbsz:1 blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:3
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:7], v[0:15] blgp:4
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] cbsz:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:3
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:7], v[4:11], v[0:15] cbsz:4
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3]
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] blgp:3
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:7], v[0:3] blgp:4
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3] cbsz:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:3
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:7], v[4:11], v[0:3] cbsz:4
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:11], v[0:3] cbsz:2 blgp:1
+# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:9], v[0:3] cbsz:1 blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:9], v[0:15] blgp:3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:7], v[0:15] blgp:4
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] cbsz:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:7], v[4:11], v[0:15] cbsz:4
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15] cbsz:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15] blgp:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3], v5, v5 op_sel_hi:[0,0,0]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:11], v[4:11], v[0:3], v5, v5 op_sel_hi:[0,0,0] blgp:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[4:9], v[4:9], v[0:3], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:2
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:9], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:2
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:11], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:11], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[4:9], v[4:9], v[0:15], v5, v5 op_sel_hi:[0,0,0] cbsz:2 blgp:2
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_f16 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_f16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_f16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf16 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf16 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x128_i8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x64_i8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x64_i8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_bf8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_bf8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_fp8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x64_fp8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_bf8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_bf8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_fp8_bf8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x64_fp8_fp8 v[10:25], a[2:5], v[4:11], v3 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3]
@@ -254,54 +254,54 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - 16.00 - - v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_f16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_bf16 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 v[0:15], v[4:5], v[6:7], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x8_bf16 a[0:15], v[4:5], v[6:7], a[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 v[0:3], v[4:5], v[6:7], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_i32_16x16x32_i8 a[0:3], v[4:5], v[6:7], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_32x32x16_i8 a[0:15], v[2:3], v[4:5], a[0:15]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_f16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_f16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[0:1], v[2:3], v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[0:1], v[2:3], a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[2:3], v[4:5], v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[4:5], a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[0:1], v[2:3], v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[0:1], v[2:3], a[34:65]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x1_4b_f32 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 v[0:31], v0, v1, v[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[34:65] blgp:7
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 v[0:3], v0, v1, v[2:5]
# CHECK-NEXT: - - - - - - 2.00 v_mfma_i32_4x4x4_16b_i8 a[0:3], v0, v1, a[2:5]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 v[0:15], v0, v1, v[18:33]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_i32_16x16x4_4b_i8 a[0:15], v0, v1, a[18:33]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 v[0:31], v0, v1, v[34:65]
# CHECK-NEXT: - - - - - - 16.00 v_mfma_i32_32x32x4_2b_i8 a[0:31], v0, v1, a[34:65]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 v[10:13], a[2:3], v[4:7], v0 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_f16 a[10:13], v[2:3], a[4:7], v1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 v[10:25], a[2:3], v[4:7], v2 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x16_f16 a[10:25], v[2:3], a[4:7], v3
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 v[10:13], a[2:3], v[4:7], v4 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x32_bf16 a[10:13], v[2:3], a[4:7], v5
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 v[10:13], a[2:3], v[4:7], v8 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_i32_16x16x64_i8 a[10:13], v[2:3], a[4:7], v9
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 v[10:25], a[2:3], v[4:7], v10 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_i32_32x32x32_i8 a[10:25], v[2:3], a[4:7], v11
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_bf8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
@@ -310,15 +310,15 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 v[0:3], v[2:3], v[4:5], v[0:3]
# CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
-# CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_bf8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_bf8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_mfma_f32_32x32x16_fp8_fp8 v[0:15], v[2:3], v[4:5], v[0:15]
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_bf8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_bf8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
# CHECK-NEXT: - - - - - - 4.00 v_smfmac_f32_16x16x64_fp8_fp8 v[0:3], a[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
-# CHECK-NEXT: - - - - - - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
+# CHECK-NEXT: - - - - 2.00 - 8.00 v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1
More information about the llvm-commits
mailing list