[llvm] 968e34e - AMDGPU/GlobalISel: RegBankLegalize AGPR support and gfx908 MFMA rules (#192603)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 22 23:01:08 PDT 2026
Author: vangthao95
Date: 2026-04-22T23:01:03-07:00
New Revision: 968e34e09ab5bb859b9a4456477224f283d19f7e
URL: https://github.com/llvm/llvm-project/commit/968e34e09ab5bb859b9a4456477224f283d19f7e
DIFF: https://github.com/llvm/llvm-project/commit/968e34e09ab5bb859b9a4456477224f283d19f7e.diff
LOG: AMDGPU/GlobalISel: RegBankLegalize AGPR support and gfx908 MFMA rules (#192603)
Initial patch adding AGPR support for RegBankLegalize along with support
for gfx908 MFMA intrinsics.
There is a regression with extra COPYs that will be addressed in future
patches.
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 82f708a0af661..0782818b1c9db 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -38,6 +38,7 @@ RegBankLegalizeHelper::RegBankLegalizeHelper(
RBLRules(RBLRules), IsWave32(ST.isWave32()),
SgprRB(&RBI.getRegBank(AMDGPU::SGPRRegBankID)),
VgprRB(&RBI.getRegBank(AMDGPU::VGPRRegBankID)),
+ AgprRB(&RBI.getRegBank(AMDGPU::AGPRRegBankID)),
VccRB(&RBI.getRegBank(AMDGPU::VCCRegBankID)) {}
bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
@@ -1673,6 +1674,8 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
case Sgpr32SExt:
case Sgpr32ZExt:
return SgprRB;
+ case AgprAnyTy:
+ return AgprRB;
case Vgpr16:
case Vgpr32:
case Vgpr64:
@@ -1703,6 +1706,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
case VgprB256:
case VgprB512:
case VgprBRC:
+ case VgprAnyTy:
case Vgpr32AExt:
case Vgpr32SExt:
case Vgpr32ZExt:
@@ -1791,6 +1795,19 @@ bool RegBankLegalizeHelper::applyMappingDst(
assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
break;
}
+ case VgprAnyTy: {
+ assert(RB == VgprRB);
+ break;
+ }
+ case AgprAnyTy: {
+ if (RB == AgprRB)
+ break;
+ Register NewAgprDst = MRI.createVirtualRegister({AgprRB, Ty});
+ Op.setReg(NewAgprDst);
+ if (!MRI.use_nodbg_empty(Reg))
+ B.buildCopy(Reg, NewAgprDst);
+ break;
+ }
// uniform in vcc/vgpr: scalars, vectors and B-types
case UniInVcc: {
assert(Ty == S1);
@@ -1983,6 +2000,20 @@ bool RegBankLegalizeHelper::applyMappingSrc(
}
break;
}
+ case VgprAnyTy: {
+ if (RB != VgprRB) {
+ auto CopyToVgpr = B.buildCopy({VgprRB, Ty}, Reg);
+ Op.setReg(CopyToVgpr.getReg(0));
+ }
+ break;
+ }
+ case AgprAnyTy: {
+ if (RB != AgprRB) {
+ auto CopyToAgpr = B.buildCopy({AgprRB, Ty}, Reg);
+ Op.setReg(CopyToAgpr.getReg(0));
+ }
+ break;
+ }
// sgpr waterfall, scalars, and vectors
case Sgpr32_WF:
case SgprV4S32_WF: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index e4aeabc3ab77b..1f9708d5c784c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -48,6 +48,7 @@ class RegBankLegalizeHelper {
const bool IsWave32;
const RegisterBank *SgprRB;
const RegisterBank *VgprRB;
+ const RegisterBank *AgprRB;
const RegisterBank *VccRB;
static constexpr LLT S1 = LLT::scalar(1);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index f4fc15262a711..77802f5f4349a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -98,6 +98,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
return MRI.getType(Reg).getSizeInBits() == 256;
case B512:
return MRI.getType(Reg).getSizeInBits() == 512;
+ case DivAnyTy:
+ return MUI.isDivergent(Reg);
case UniS1:
return MRI.getType(Reg) == LLT::scalar(1) && MUI.isUniform(Reg);
case UniS16:
@@ -2029,6 +2031,22 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr32}})
.Div(S16, {{Vgpr16}, {IntrId, Vgpr32}});
+ // TODO: Add handling for GFX90A+ which should use VGPRs instead of AGPRs.
+ bool HasGFX90AInsts = ST->hasGFX90AInsts();
+ addRulesForIOpcs({amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
+ amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
+ amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
+ amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
+ amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
+ amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
+ amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
+ amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
+ amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
+ amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
+ .Any({{DivAnyTy},
+ {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
+ !HasGFX90AInsts);
+
// WMMA/SWMMAC intrinsics: all register operands map to VGPR.
addRulesForIOpcs(
{// WMMA GFX11+
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index ec5f67bf62b0c..35bbefb54d29e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -58,6 +58,9 @@ enum UniformityLLTOpPredicateID {
DivS64,
DivS128,
+ // any LLT, divergent-check only predicate
+ DivAnyTy,
+
// pointers
P0,
P1,
@@ -149,6 +152,10 @@ enum RegBankLLTMappingApplyID {
Imm,
Vcc,
+ // any LLT, bank-only apply IDs
+ VgprAnyTy,
+ AgprAnyTy,
+
// sgpr scalars, pointers, vectors and B-types
Sgpr16,
Sgpr32,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
index e34a62d29a102..06cdc075da42c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass='amdgpu-regbankselect,amdgpu-regbanklegalize' -verify-machineinstrs %s -o - | FileCheck %s
---
name: mfma_f32_32x32x1f32_vva
@@ -15,9 +14,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -42,8 +43,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
@@ -64,9 +66,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -91,8 +95,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -113,9 +118,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -140,8 +147,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -162,9 +170,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -189,8 +199,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -211,9 +222,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -238,8 +251,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -260,9 +274,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s16>) = COPY $vgpr2_vgpr3
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -287,8 +303,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>)
%0:_(<4 x s16>) = COPY $sgpr32_sgpr33
%1:_(<4 x s16>) = COPY $sgpr34_sgpr35
%2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
@@ -309,9 +326,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s16>) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -336,8 +355,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(<4 x s16>) = COPY $sgpr32_sgpr33
%1:_(<4 x s16>) = COPY $sgpr34_sgpr35
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -358,9 +378,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s16>) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -385,8 +407,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(<4 x s16>) = COPY $sgpr32_sgpr33
%1:_(<4 x s16>) = COPY $sgpr34_sgpr35
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -407,9 +430,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s16>) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -434,8 +459,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(<4 x s16>) = COPY $sgpr32_sgpr33
%1:_(<4 x s16>) = COPY $sgpr34_sgpr35
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -456,9 +482,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(<4 x s16>) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -483,8 +511,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(<4 x s16>) = COPY $sgpr32_sgpr33
%1:_(<4 x s16>) = COPY $sgpr34_sgpr35
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -505,9 +534,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr2
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -532,8 +563,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
@@ -554,9 +586,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr2
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -581,8 +615,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -603,9 +638,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr2
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -630,8 +667,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -652,9 +690,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr2
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -679,8 +719,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -701,9 +742,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr2
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -728,8 +771,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(s32) = COPY $sgpr32
%1:_(s32) = COPY $sgpr33
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -750,9 +794,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr2
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -777,8 +823,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>)
%0:_(<2 x s16>) = COPY $sgpr32
%1:_(<2 x s16>) = COPY $sgpr33
%2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
@@ -799,9 +846,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr2
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -826,8 +875,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(<2 x s16>) = COPY $sgpr32
%1:_(<2 x s16>) = COPY $sgpr33
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -848,9 +898,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr2
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -875,8 +927,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(<2 x s16>) = COPY $sgpr32
%1:_(<2 x s16>) = COPY $sgpr33
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
@@ -897,9 +950,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr2
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -924,8 +979,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>)
%0:_(<2 x s16>) = COPY $sgpr32
%1:_(<2 x s16>) = COPY $sgpr33
%2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -946,9 +1002,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(<2 x s16>) = COPY $vgpr2
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -973,8 +1031,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
- ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>)
%0:_(<2 x s16>) = COPY $sgpr32
%1:_(<2 x s16>) = COPY $sgpr33
%2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
index f0040b61efaae..45e5755675d83 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefix=GFX908 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefix=GFX908 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908,GFX908-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908,GFX908-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GFX908,GFX908-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GFX908,GFX908-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefix=GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GFX90A-VGPR %s
@@ -12,140 +14,260 @@ declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16>, <2 x i16>, <4 x
declare i32 @llvm.amdgcn.workitem.id.x()
define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 {
-; GFX908-LABEL: test_mfma_f32_32x32x2bf16:
-; GFX908: ; %bb.0: ; %bb
-; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v4, 0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
-; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v0, s16
-; GFX908-NEXT: v_mov_b32_e32 v1, s17
-; GFX908-NEXT: v_mov_b32_e32 v2, s18
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s21
-; GFX908-NEXT: v_mov_b32_e32 v1, s22
-; GFX908-NEXT: v_mov_b32_e32 v2, s23
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s24
-; GFX908-NEXT: v_mov_b32_e32 v1, s25
-; GFX908-NEXT: v_mov_b32_e32 v2, s26
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s27
-; GFX908-NEXT: v_mov_b32_e32 v1, s28
-; GFX908-NEXT: v_mov_b32_e32 v2, s29
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s30
-; GFX908-NEXT: v_mov_b32_e32 v1, s31
-; GFX908-NEXT: v_mov_b32_e32 v2, s0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s1
-; GFX908-NEXT: v_mov_b32_e32 v1, s2
-; GFX908-NEXT: v_mov_b32_e32 v2, s3
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s4
-; GFX908-NEXT: v_mov_b32_e32 v1, s5
-; GFX908-NEXT: v_mov_b32_e32 v2, s6
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s7
-; GFX908-NEXT: v_mov_b32_e32 v1, s8
-; GFX908-NEXT: v_mov_b32_e32 v2, s9
-; GFX908-NEXT: v_mov_b32_e32 v3, s19
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s10
-; GFX908-NEXT: v_mov_b32_e32 v1, s11
-; GFX908-NEXT: v_mov_b32_e32 v2, s12
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, s13
-; GFX908-NEXT: v_mov_b32_e32 v1, s14
-; GFX908-NEXT: v_mov_b32_e32 v2, s15
-; GFX908-NEXT: v_mov_b32_e32 v3, 1
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v5
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 2
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
-; GFX908-NEXT: s_nop 15
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a28
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a16
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a20
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a8
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
-; GFX908-NEXT: s_endpgm
+; GFX908-SDAG-LABEL: test_mfma_f32_32x32x2bf16:
+; GFX908-SDAG: ; %bb.0: ; %bb
+; GFX908-SDAG-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0
+; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s16
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s18
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s21
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s22
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s23
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s24
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s25
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s26
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s27
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s28
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s29
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s30
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s31
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a16, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a17, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a18, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a19, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a20, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a21, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a22, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s7
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s8
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s9
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, s19
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a23, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a24, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a25, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s10
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s11
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s20
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a26, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a27, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a28, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s15
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, 1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v5
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a29, v0
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a30, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a31, v2
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 2
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-SDAG-NEXT: s_nop 15
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a27
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a24
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a28
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a16
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a20
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a8
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a4
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
+; GFX908-SDAG-NEXT: s_endpgm
+;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2bf16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x2bf16:
; GFX90A: ; %bb.0: ; %bb
@@ -268,72 +390,138 @@ bb:
}
define amdgpu_kernel void @test_mfma_f32_16x16x2bf16(ptr addrspace(1) %arg) #0 {
-; GFX908-LABEL: test_mfma_f32_16x16x2bf16:
-; GFX908: ; %bb.0: ; %bb
-; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v0, 1
-; GFX908-NEXT: v_mov_b32_e32 v12, 0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v13, s0
-; GFX908-NEXT: v_mov_b32_e32 v1, s1
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v13
-; GFX908-NEXT: v_mov_b32_e32 v13, s3
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v13
-; GFX908-NEXT: v_mov_b32_e32 v1, s4
-; GFX908-NEXT: v_mov_b32_e32 v2, s5
-; GFX908-NEXT: v_mov_b32_e32 v13, s6
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v13
-; GFX908-NEXT: v_mov_b32_e32 v1, s7
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: v_mov_b32_e32 v13, s9
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v13
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
-; GFX908-NEXT: v_mov_b32_e32 v2, s11
-; GFX908-NEXT: v_mov_b32_e32 v13, s12
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v13
-; GFX908-NEXT: v_mov_b32_e32 v1, s13
-; GFX908-NEXT: v_mov_b32_e32 v2, s14
-; GFX908-NEXT: v_mov_b32_e32 v13, s15
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v13
-; GFX908-NEXT: v_mov_b32_e32 v1, 2
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX908-NEXT: s_nop 9
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a4
-; GFX908-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
-; GFX908-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
-; GFX908-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
-; GFX908-NEXT: s_endpgm
+; GFX908-SDAG-LABEL: test_mfma_f32_16x16x2bf16:
+; GFX908-SDAG: ; %bb.0: ; %bb
+; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v12, 0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s0
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s3
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s6
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s9
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s12
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s15
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-SDAG-NEXT: s_nop 9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4
+; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48
+; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32
+; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
+; GFX908-SDAG-NEXT: s_endpgm
+;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x2bf16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_16x16x2bf16:
; GFX90A: ; %bb.0: ; %bb
@@ -477,72 +665,139 @@ bb:
}
define amdgpu_kernel void @test_mfma_f32_32x32x4bf16(ptr addrspace(1) %arg) #0 {
-; GFX908-LABEL: test_mfma_f32_32x32x4bf16:
-; GFX908: ; %bb.0: ; %bb
-; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v0, 1
-; GFX908-NEXT: v_mov_b32_e32 v16, 0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v17, s0
-; GFX908-NEXT: v_mov_b32_e32 v1, s1
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v17
-; GFX908-NEXT: v_mov_b32_e32 v17, s3
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s4
-; GFX908-NEXT: v_mov_b32_e32 v2, s5
-; GFX908-NEXT: v_mov_b32_e32 v17, s6
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s7
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: v_mov_b32_e32 v17, s9
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
-; GFX908-NEXT: v_mov_b32_e32 v2, s11
-; GFX908-NEXT: v_mov_b32_e32 v17, s12
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s13
-; GFX908-NEXT: v_mov_b32_e32 v2, s14
-; GFX908-NEXT: v_mov_b32_e32 v17, s15
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, 2
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX908-NEXT: s_nop 15
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a0
-; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
-; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
-; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
-; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
-; GFX908-NEXT: s_endpgm
+; GFX908-SDAG-LABEL: test_mfma_f32_32x32x4bf16:
+; GFX908-SDAG: ; %bb.0: ; %bb
+; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-SDAG-NEXT: s_nop 15
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a0
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
+; GFX908-SDAG-NEXT: s_endpgm
+;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4bf16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x4bf16:
; GFX90A: ; %bb.0: ; %bb
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 61593a8a81a7c..4048f446371ef 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR %s
@@ -8,73 +10,140 @@ declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32, i32, <16 x i32>, i32, i3
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32, i32, <4 x i32>, i32, i32, i32)
define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
-; GFX908-LABEL: test_mfma_i32_32x32x8i8:
-; GFX908: ; %bb.0: ; %bb
-; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v0, 1
-; GFX908-NEXT: v_mov_b32_e32 v16, 0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v17, s0
-; GFX908-NEXT: v_mov_b32_e32 v1, s1
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v17
-; GFX908-NEXT: v_mov_b32_e32 v17, s3
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s4
-; GFX908-NEXT: v_mov_b32_e32 v2, s5
-; GFX908-NEXT: v_mov_b32_e32 v17, s6
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s7
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: v_mov_b32_e32 v17, s9
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
-; GFX908-NEXT: v_mov_b32_e32 v2, s11
-; GFX908-NEXT: v_mov_b32_e32 v17, s12
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, s13
-; GFX908-NEXT: v_mov_b32_e32 v2, s14
-; GFX908-NEXT: v_mov_b32_e32 v17, s15
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v17
-; GFX908-NEXT: v_mov_b32_e32 v1, 2
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX908-NEXT: s_nop 15
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a8
-; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX908-NEXT: s_endpgm
+; GFX908-SDAG-LABEL: test_mfma_i32_32x32x8i8:
+; GFX908-SDAG: ; %bb.0: ; %bb
+; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-SDAG-NEXT: s_nop 15
+; GFX908-SDAG-NEXT: s_nop 1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-SDAG-NEXT: s_nop 0
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-SDAG-NEXT: s_endpgm
+;
+; GFX908-GISEL-LABEL: test_mfma_i32_32x32x8i8:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_i32_32x32x8i8:
; GFX90A: ; %bb.0: ; %bb
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 0161cdf03deac..8152b5e38477c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC,GFX908,GFX908_A %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC,GFX908,GFX908_A %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-NOLIT-SRCC %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-LIT-SRCC %s
; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
@@ -156,6 +158,126 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -533,6 +655,72 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -713,6 +901,34 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -866,6 +1082,73 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x2f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x2f32:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1049,6 +1332,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x4f32 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x4f32:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1274,6 +1585,129 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s40
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s16
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s19
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s22
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s23
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s25
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s26
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s28
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s41
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s29
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s31
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s42
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s43
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[0:1], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x4f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -1673,6 +2107,75 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s21
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s22
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s23
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x4f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x4f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -1871,6 +2374,37 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x4f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x4f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2042,6 +2576,76 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s21
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s22
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s23
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2241,6 +2845,37 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x16f16:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2463,6 +3098,126 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_i32_32x32x4i8:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -2826,6 +3581,72 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -3025,6 +3846,53 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 9
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1
@@ -3056,6 +3924,37 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 9
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm
+;
; GFX90A-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: v_mov_b32_e32 v16, 1
@@ -3139,6 +4038,34 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -3247,6 +4174,28 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1
@@ -3265,6 +4214,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 3
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 0
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm
+;
; GFX90A-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: v_mov_b32_e32 v0, 1
@@ -3334,6 +4300,33 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX908-GISEL: ; %bb.0:
+; GFX908-GISEL-NEXT: s_movk_i32 s2, 0x41
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; LIT-SRCC: ; %bb.0:
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x41
@@ -3550,6 +4543,127 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -3932,6 +5046,73 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -4117,6 +5298,35 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3]
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3]
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -4232,6 +5442,28 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
@@ -4250,6 +5482,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, 1.0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 3
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 0
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm
+;
; GFX90A-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: v_mov_b32_e32 v0, 1.0
@@ -4349,6 +5598,53 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 9
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
@@ -4383,6 +5679,37 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, 1.0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 9
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm
+;
; GFX90A-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: v_mov_b32_e32 v16, 1.0
@@ -4494,6 +5821,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s1, s0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s2, 0x40004000
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s3, s2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 15
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x3c003c00
@@ -4530,6 +5911,44 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: global_store_dwordx4 v13, v[9:12], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s1, s0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, s0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, s1
+; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s0, 0x40004000
+; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s1, s0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v2, s0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], 1.0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 15
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:16
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:32
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:48
+; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm
+;
; GFX90A-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -4690,6 +6109,90 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 15
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
@@ -4746,6 +6249,58 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1]
; LIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, 0
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 15
+; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm
+;
; GFX90A-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: v_mov_b32_e32 v32, 1.0
@@ -4835,6 +6390,28 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, 2.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, 1.0
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3]
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
@@ -4962,6 +6539,53 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, 2.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15]
+; GFX908-GISEL-NEXT: s_nop 9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
@@ -5208,6 +6832,90 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, 0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
@@ -5537,6 +7245,33 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_mov_b32 s2, 0x42f60000
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2
+; GFX908-GISEL-NEXT: v_lshlrev_b32_e32 v4, 4, v0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3]
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 0x42f60000
@@ -5646,6 +7381,33 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_mov_b32 s2, 0x42f60000
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3]
+; GFX908-GISEL-NEXT: s_nop 3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: s_nop 0
+; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x42f60000
@@ -5824,6 +7586,101 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
+; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX908-GISEL: ; %bb.0: ; %bb
+; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX908-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX908-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v0
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v5
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v8
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v11
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v14
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v15
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] offset:64
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:80
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
+; GFX908-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:112
+; GFX908-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v15
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v14
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v13
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v12
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v11
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v10
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v9
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v8
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v7
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v6
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v5
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v4
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v3
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v2
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v1
+; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v0
+; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 2.0
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v16, v0, a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-GISEL-NEXT: s_nop 15
+; GFX908-GISEL-NEXT: s_nop 1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX908-GISEL-NEXT: s_endpgm
+;
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
More information about the llvm-commits
mailing list