[llvm] ca42466 - AMDGPU/GlobalISel: RegBankLegalize rules for gfx90a/gfx942 MFMAs (#194076)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 15:47:14 PDT 2026
Author: vangthao95
Date: 2026-06-10T15:47:09-07:00
New Revision: ca4246669006ec9d6bc88ab2acb03f308a07d422
URL: https://github.com/llvm/llvm-project/commit/ca4246669006ec9d6bc88ab2acb03f308a07d422
DIFF: https://github.com/llvm/llvm-project/commit/ca4246669006ec9d6bc88ab2acb03f308a07d422.diff
LOG: AMDGPU/GlobalISel: RegBankLegalize rules for gfx90a/gfx942 MFMAs (#194076)
Add rules for gfx90a/gfx942 MFMA/SMFMAC intrinsics.
I see some regressions with imm splat tests and stores that could have
taken agprs. I will try to address those in a follow-up patch.
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index d89e6455930a0..07e71b24c59f9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -35,9 +35,9 @@ using namespace AMDGPU;
RegBankLegalizeHelper::RegBankLegalizeHelper(
MachineIRBuilder &B, const MachineUniformityInfo &MUI,
const RegisterBankInfo &RBI, const RegBankLegalizeRules &RBLRules)
- : MF(B.getMF()), ST(MF.getSubtarget<GCNSubtarget>()), B(B),
- MRI(*B.getMRI()), MUI(MUI), RBI(RBI), MORE(MF, nullptr),
- RBLRules(RBLRules), IsWave32(ST.isWave32()),
+ : MF(B.getMF()), MFI(MF.getInfo<SIMachineFunctionInfo>()),
+ ST(MF.getSubtarget<GCNSubtarget>()), B(B), MRI(*B.getMRI()), MUI(MUI),
+ RBI(RBI), MORE(MF, nullptr), RBLRules(RBLRules), IsWave32(ST.isWave32()),
SgprRB(&RBI.getRegBank(AMDGPU::SGPRRegBankID)),
VgprRB(&RBI.getRegBank(AMDGPU::VGPRRegBankID)),
AgprRB(&RBI.getRegBank(AMDGPU::AGPRRegBankID)),
@@ -1900,6 +1900,18 @@ bool RegBankLegalizeHelper::applyMappingDst(
B.buildCopy(Reg, NewAgprDst);
break;
}
+ case VgprOrAgprAnyTy: {
+ const unsigned NumRegs = Ty.getSizeInBits() / 32;
+ const RegisterBank *DstRB =
+ MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
+ if (RB == DstRB)
+ break;
+ Register NewDst = MRI.createVirtualRegister({DstRB, Ty});
+ Op.setReg(NewDst);
+ if (!MRI.use_nodbg_empty(Reg))
+ B.buildCopy(Reg, NewDst);
+ break;
+ }
// uniform in vcc/vgpr: scalars, vectors and B-types
case UniInVcc: {
assert(Ty == S1);
@@ -2117,6 +2129,14 @@ bool RegBankLegalizeHelper::applyMappingSrc(
}
break;
}
+ case VgprOrAgprAnyTy: {
+ const unsigned NumRegs = Ty.getSizeInBits() / 32;
+ const RegisterBank *SrcRB =
+ MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
+ if (RB != SrcRB)
+ Op.setReg(B.buildCopy({SrcRB, Ty}, Reg).getReg(0));
+ break;
+ }
// sgpr waterfall, scalars, and vectors
case Sgpr32_WF:
case SgprV4S32_WF: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index 1f9708d5c784c..113360c0949ca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -18,6 +18,7 @@
namespace llvm {
class MachineIRBuilder;
+class SIMachineFunctionInfo;
namespace AMDGPU {
@@ -38,6 +39,7 @@ struct WaterfallInfo {
// instruction(s).
class RegBankLegalizeHelper {
MachineFunction &MF;
+ const SIMachineFunctionInfo *MFI;
const GCNSubtarget &ST;
MachineIRBuilder &B;
MachineRegisterInfo &MRI;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7161ec84a038e..7dd2604a3ec44 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -2457,8 +2457,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr32}})
.Div(S16, {{Vgpr16}, {IntrId, Vgpr32}});
- // TODO: Add handling for GFX90A+ which should use VGPRs instead of AGPRs.
bool HasGFX90AInsts = ST->hasGFX90AInsts();
+
+ // On gfx90a+ both AGPR-form and VGPR-form exists
addRulesForIOpcs({amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
@@ -2471,7 +2472,39 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
.Any({{DivAnyTy},
{{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
- !HasGFX90AInsts);
+ !HasGFX90AInsts)
+ .Any({{DivAnyTy},
+ {{VgprOrAgprAnyTy},
+ {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}},
+ HasGFX90AInsts);
+
+ // gfx90a+ only MFMAs
+ addRulesForIOpcs(
+ {amdgcn_mfma_f32_32x32x4bf16_1k, amdgcn_mfma_f32_16x16x4bf16_1k,
+ amdgcn_mfma_f32_4x4x4bf16_1k, amdgcn_mfma_f32_32x32x8bf16_1k,
+ amdgcn_mfma_f32_16x16x16bf16_1k, amdgcn_mfma_f64_16x16x4f64,
+ amdgcn_mfma_f64_4x4x4f64, amdgcn_mfma_i32_16x16x32_i8,
+ amdgcn_mfma_i32_32x32x16_i8, amdgcn_mfma_f32_16x16x8_xf32,
+ amdgcn_mfma_f32_32x32x4_xf32, amdgcn_mfma_f32_16x16x32_bf8_bf8,
+ amdgcn_mfma_f32_16x16x32_bf8_fp8, amdgcn_mfma_f32_16x16x32_fp8_bf8,
+ amdgcn_mfma_f32_16x16x32_fp8_fp8, amdgcn_mfma_f32_32x32x16_bf8_bf8,
+ amdgcn_mfma_f32_32x32x16_bf8_fp8, amdgcn_mfma_f32_32x32x16_fp8_bf8,
+ amdgcn_mfma_f32_32x32x16_fp8_fp8})
+ .Any({{DivAnyTy},
+ {{VgprOrAgprAnyTy},
+ {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}});
+
+ addRulesForIOpcs(
+ {amdgcn_smfmac_f32_16x16x32_f16, amdgcn_smfmac_f32_32x32x16_f16,
+ amdgcn_smfmac_f32_16x16x32_bf16, amdgcn_smfmac_f32_32x32x16_bf16,
+ amdgcn_smfmac_i32_16x16x64_i8, amdgcn_smfmac_i32_32x32x32_i8,
+ amdgcn_smfmac_f32_16x16x64_bf8_bf8, amdgcn_smfmac_f32_16x16x64_bf8_fp8,
+ amdgcn_smfmac_f32_16x16x64_fp8_bf8, amdgcn_smfmac_f32_16x16x64_fp8_fp8,
+ amdgcn_smfmac_f32_32x32x32_bf8_bf8, amdgcn_smfmac_f32_32x32x32_bf8_fp8,
+ amdgcn_smfmac_f32_32x32x32_fp8_bf8, amdgcn_smfmac_f32_32x32x32_fp8_fp8})
+ .Any({{DivAnyTy},
+ {{VgprOrAgprAnyTy},
+ {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, VgprAnyTy}}});
// WMMA/SWMMAC intrinsics: all register operands map to VGPR.
addRulesForIOpcs(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index a7f3aafe098fe..147e80877ec6e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -170,6 +170,7 @@ enum RegBankLLTMappingApplyID {
// any LLT, bank-only apply IDs
VgprAnyTy,
AgprAnyTy,
+ VgprOrAgprAnyTy,
// sgpr scalars, pointers, vectors and B-types
Sgpr16,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
index 5c45565b4d047..86bcac99a22a7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -verify-machineinstrs -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
index 5f040779dd42d..ed24ab123584d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect -regbankselect-fast -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect -regbankselect-greedy -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass='amdgpu-regbankselect,amdgpu-regbanklegalize' -amdgpu-mfma-vgpr-form=0 %s -o - | FileCheck %s
---
name: mfma_f32_32x32x4bf16_1k_vva
@@ -10,22 +9,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-LABEL: name: mfma_f32_32x32x4bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_32x32x4bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_32x32x4bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -41,22 +34,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-LABEL: name: mfma_f32_16x16x4bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_16x16x4bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_16x16x4bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -72,22 +59,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_f32_4x4x4bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_4x4x4bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_4x4x4bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -103,22 +84,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-LABEL: name: mfma_f32_32x32x8bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_32x32x8bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_32x32x8bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -134,22 +109,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_f32_16x16x16bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_16x16x16bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_16x16x16bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -165,22 +134,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; FAST-LABEL: name: mfma_f64_16x16x4f64_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<8 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.16x16x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<8 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INT]](<8 x s32>)
- ; GREEDY-LABEL: name: mfma_f64_16x16x4f64_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<8 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.16x16x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<8 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INT]](<8 x s32>)
+ ; CHECK-LABEL: name: mfma_f64_16x16x4f64_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<8 x s32>) = COPY [[COPY2]](<8 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<8 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.16x16x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<8 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<8 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<8 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[COPY4]](<8 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
@@ -196,22 +159,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
- ; FAST-LABEL: name: mfma_f64_4x4x4f64_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<2 x s32>) = COPY $agpr0_agpr1
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<2 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.4x4x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<2 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1 = COPY [[INT]](<2 x s32>)
- ; GREEDY-LABEL: name: mfma_f64_4x4x4f64_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<2 x s32>) = COPY $agpr0_agpr1
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<2 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.4x4x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<2 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1 = COPY [[INT]](<2 x s32>)
+ ; CHECK-LABEL: name: mfma_f64_4x4x4f64_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<2 x s32>) = COPY $agpr0_agpr1
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<2 x s32>) = COPY [[COPY2]](<2 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<2 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.4x4x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<2 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<2 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY4]](<2 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<2 x s32>) = COPY $agpr0_agpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
index 12208c133a306..120ec7912496a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=regbankselect -regbankselect-fast -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=regbankselect -regbankselect-greedy -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass='amdgpu-regbankselect,amdgpu-regbanklegalize' -amdgpu-mfma-vgpr-form=0 %s -o - | FileCheck %s
---
name: mfma_i32_16x16x32_i8_vva
@@ -10,22 +9,16 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_i32_16x16x32_i8_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x32.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_i32_16x16x32_i8_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x32.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_i32_16x16x32_i8_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x32.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -41,22 +34,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-LABEL: name: mfma_i32_32x32x16_i8_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x16.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: mfma_i32_32x32x16_i8_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x16.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: mfma_i32_32x32x16_i8_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x16.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -72,22 +59,16 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_f32_16x16x8_xf32_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_16x16x8_xf32_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_16x16x8_xf32_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -103,22 +84,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-LABEL: name: mfma_f32_32x32x4_xf32_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_32x32x4_xf32_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_32x32x4_xf32_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -134,24 +109,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_16x16x32_f16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_16x16x32_f16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_16x16x32_f16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<4 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY5]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -168,24 +136,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_32x32x16_f16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_32x32x16_f16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_32x32x16_f16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<16 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY5]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -202,24 +163,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_16x16x32_bf16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_16x16x32_bf16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_16x16x32_bf16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<4 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY5]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -236,24 +190,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_32x32x16_bf16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_32x32x16_bf16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_32x32x16_bf16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<16 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY5]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -270,24 +217,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-LABEL: name: smfmac_i32_16x16x64_i8_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.16x16x64.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: smfmac_i32_16x16x64_i8_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.16x16x64.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: smfmac_i32_16x16x64_i8_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.16x16x64.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<4 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY5]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -304,24 +244,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-LABEL: name: smfmac_i32_32x32x32_i8_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.32x32x32.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: smfmac_i32_32x32x32_i8_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.32x32x32.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: smfmac_i32_32x32x32_i8_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.32x32x32.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<16 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY5]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
index e6379bd4dbb15..835d11642f009 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
@@ -1,8 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX90A-VGPR %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX942-VGPR %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-VGPR %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-VGPR-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-VGPR %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-VGPR-GISEL %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32)
@@ -71,6 +75,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x4bf16_1k v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -128,6 +174,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
+; GFX942-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[36:37]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -185,6 +273,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4bf16_1k v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -241,6 +371,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -278,6 +450,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -305,6 +505,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -332,6 +560,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 9
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -358,6 +614,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -386,6 +670,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x4bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -404,6 +707,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -422,6 +744,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 3
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -439,6 +780,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -477,6 +837,35 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -504,6 +893,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -532,6 +949,35 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -558,6 +1004,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -586,6 +1060,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x16bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -604,6 +1097,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 5
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -622,6 +1134,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 9
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -639,6 +1170,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX942-VGPR-NEXT: s_nop 6
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 5
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -665,6 +1215,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-GISEL-NEXT: s_nop 7
+; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_4x4x4f64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -681,6 +1247,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-GISEL-NEXT: s_nop 7
+; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_4x4x4f64:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -697,6 +1279,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX90A-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 3
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 7
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_4x4x4f64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -712,6 +1310,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX942-VGPR-NEXT: s_nop 7
; GFX942-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 7
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double 0.0, i32 0, i32 0, i32 0)
%mai.2 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double %mai.1, i32 1, i32 2, i32 3)
@@ -743,6 +1357,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
@@ -766,6 +1402,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
@@ -789,6 +1447,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
@@ -811,6 +1491,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x double>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %in.1, i32 1, i32 2, i32 3)
@@ -836,6 +1538,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -853,6 +1572,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -870,6 +1606,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -886,6 +1639,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> zeroinitializer, i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -911,6 +1681,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -928,6 +1715,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -945,6 +1749,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -961,6 +1782,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 -1 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -986,6 +1824,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1003,6 +1866,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1020,6 +1908,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1036,6 +1949,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double 1.0), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1061,6 +1999,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1078,6 +2041,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1095,6 +2083,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1111,6 +2124,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double -1.0), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1136,6 +2174,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1153,6 +2216,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1170,6 +2258,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1186,6 +2299,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 64 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1222,6 +2360,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX90A-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1250,6 +2414,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1278,6 +2468,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1305,6 +2521,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877906944 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1338,6 +2580,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1363,6 +2631,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1388,6 +2682,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1412,6 +2732,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877907008 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1445,6 +2791,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1470,6 +2842,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1495,6 +2893,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1519,6 +2943,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (<2 x float> splat (float 1.0) to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1555,6 +3005,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX90A-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1583,6 +3057,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1611,6 +3109,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1638,6 +3160,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 0.0, double 0.0, double 0.0, double 1.0>, i32 0, i32 0, i32 0)
store <4 x double> %mai.1, ptr addrspace(1) %arg
@@ -1673,6 +3219,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1701,6 +3272,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1729,6 +3325,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1756,6 +3377,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 123.0, double 123.0, double 123.0, double 123.0>, i32 0, i32 0, i32 0)
store <4 x double> %mai.1, ptr addrspace(1) %arg
@@ -1763,6 +3409,3 @@ bb:
}
attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GCN: {{.*}}
-; VGPR: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
index 700fc6db8549f..f161e84512f25 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-SDAG,GFX942-VGPRCD-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-SDAG,GFX950-VGPRCD-SDAG %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-GISEL,GFX950-VGPRCD-GISEL %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 4048f446371ef..f380634090348 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -3,8 +3,10 @@
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GFX90A-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GFX90A-VGPR-GISEL %s
declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32, i32, <16 x i32>, i32, i32, i32)
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32, i32, <4 x i32>, i32, i32, i32)
@@ -145,67 +147,145 @@ define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX908-GISEL-NEXT: s_endpgm
;
-; GFX90A-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A: ; %bb.0: ; %bb
-; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v1, 2
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4
-; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5
-; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6
-; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7
-; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8
-; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9
-; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10
-; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11
-; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12
-; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13
-; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14
-; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15
-; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-NEXT: s_nop 15
-; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
-; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
-; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
-; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
-; GFX90A-NEXT: s_endpgm
+; GFX90A-SDAG-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-SDAG: ; %bb.0: ; %bb
+; GFX90A-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a4, s4
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a5, s5
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a6, s6
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a7, s7
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a8, s8
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a9, s9
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a10, s10
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a11, s11
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a12, s12
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a13, s13
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a14, s14
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a15, s15
+; GFX90A-SDAG-NEXT: s_nop 1
+; GFX90A-SDAG-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-SDAG-NEXT: s_nop 15
+; GFX90A-SDAG-NEXT: s_nop 1
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
+; GFX90A-SDAG-NEXT: s_endpgm
;
-; GFX90A-VGPR-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A-VGPR: ; %bb.0: ; %bb
-; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-VGPR-NEXT: s_nop 15
-; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-VGPR-NEXT: s_endpgm
+; GFX90A-GISEL-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a4, s4
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a5, s5
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a6, s6
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a7, s7
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a8, s8
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a9, s9
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a10, s10
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a11, s11
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a12, s12
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a13, s13
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a14, s14
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a15, s15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
+; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-VGPR-SDAG: ; %bb.0: ; %bb
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: s_nop 1
+; GFX90A-VGPR-SDAG-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-SDAG-NEXT: s_nop 15
+; GFX90A-VGPR-SDAG-NEXT: s_nop 1
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-SDAG-NEXT: s_endpgm
+;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -242,41 +322,77 @@ define amdgpu_kernel void @test_mfma_i32_16x16x16i8(ptr addrspace(1) %arg) #0 {
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX908-NEXT: s_endpgm
;
-; GFX90A-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A: ; %bb.0: ; %bb
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v2, 2
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
-; GFX90A-NEXT: s_endpgm
+; GFX90A-SDAG-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-SDAG: ; %bb.0: ; %bb
+; GFX90A-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v2, 2
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-SDAG-NEXT: s_nop 1
+; GFX90A-SDAG-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-SDAG-NEXT: s_nop 10
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
+; GFX90A-SDAG-NEXT: s_endpgm
+;
+; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
+; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-VGPR-SDAG: ; %bb.0: ; %bb
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v6, 2
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: s_nop 1
+; GFX90A-VGPR-SDAG-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-SDAG-NEXT: s_nop 10
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-VGPR-SDAG-NEXT: s_endpgm
;
-; GFX90A-VGPR-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A-VGPR: ; %bb.0: ; %bb
-; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: s_nop 10
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
-; GFX90A-VGPR-NEXT: s_endpgm
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 9
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 8152b5e38477c..5eeefe90296a0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -3,9 +3,12 @@
; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-NOLIT-SRCC %s
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s
; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-LIT-SRCC %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR-GISEL %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32)
@@ -469,6 +472,46 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -525,6 +568,46 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -580,6 +663,46 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 1, i32 2, i32 3)
@@ -814,6 +937,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -840,6 +989,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -865,6 +1040,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3)
@@ -974,6 +1175,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -991,6 +1209,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1007,6 +1242,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1243,6 +1495,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x2f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x2f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1270,6 +1549,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x2f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1296,6 +1602,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1405,6 +1738,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x4f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1422,6 +1772,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1438,6 +1805,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 9
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x4f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1907,6 +2291,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[40:41], s[40:41] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[42:43], s[42:43] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x4f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -1967,6 +2392,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[40:41]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[42:43]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -2026,6 +2492,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[40:41]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[42:43]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2275,6 +2782,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2304,6 +2838,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2332,6 +2893,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2456,6 +3044,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x4f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2476,6 +3082,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2495,6 +3119,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2746,6 +3388,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x8f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2775,6 +3445,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2803,6 +3500,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2927,6 +3651,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x16f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2947,6 +3689,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 5
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2966,6 +3726,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-VGPR-NEXT: s_nop 6
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 5
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -3395,6 +4173,46 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x4i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_32x32x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -3451,6 +4269,46 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_32x32x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -3506,6 +4364,46 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 1, i32 2, <32 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -3740,6 +4638,32 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_16x16x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -3766,6 +4690,32 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -3791,6 +4741,32 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -3971,6 +4947,46 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 64
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 8
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 1
@@ -3987,6 +5003,46 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 64
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1
@@ -4002,6 +5058,46 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> splat (i32 64), i32 1, i32 2, i32 3)
@@ -4111,6 +5207,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_4x4x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -4128,6 +5241,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -4144,6 +5274,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -4244,6 +5391,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1
@@ -4257,6 +5423,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1
@@ -4269,6 +5454,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 1), i32 1, i32 2, i32 3)
@@ -4367,6 +5571,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX90A-GISEL: ; %bb.0:
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-GISEL-NEXT: s_movk_i32 s0, 0x41
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; GFX942: ; %bb.0:
; GFX942-NEXT: v_mov_b32_e32 v5, 1
@@ -4384,6 +5607,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-GISEL-NEXT: s_movk_i32 s0, 0x41
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; GFX942-VGPR: ; %bb.0:
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1
@@ -4400,6 +5642,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX942-VGPR-GISEL: ; %bb.0:
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-VGPR-GISEL-NEXT: s_movk_i32 s0, 0x41
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 65), i32 1, i32 2, i32 3)
store <4 x i32> %mai.1, ptr addrspace(1) %arg
@@ -4857,6 +6118,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -4914,6 +6216,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -4970,6 +6313,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
@@ -5207,6 +6591,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -5234,6 +6645,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -5260,6 +6698,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 0, i32 0, i32 0)
@@ -5374,6 +6839,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -5393,6 +6876,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -5411,6 +6913,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 0, i32 0, i32 0)
@@ -5512,6 +7033,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -5525,6 +7065,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -5537,6 +7096,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <4 x float> %mai.1, ptr addrspace(1) %arg
@@ -5726,6 +7304,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 8
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 1.0
@@ -5742,6 +7360,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 7
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1.0
@@ -5757,6 +7415,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 7
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -5968,6 +7666,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, 0x40004000
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s2
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -5986,6 +7729,50 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s2, 0x40004000
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s2
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -6003,6 +7790,50 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, 0x40004000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s2
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>, <4 x half> <half 2.0, half 2.0, half 2.0, half 2.0>, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -6322,6 +8153,27 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, 0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v32, 1.0
@@ -6342,6 +8194,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 1.0
@@ -6361,6 +8233,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
store <32 x float> %mai.1, ptr addrspace(1) %arg
@@ -6449,6 +8341,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 2.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6464,6 +8375,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 2.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6478,6 +8408,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 2.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <4 x float> %mai.1, ptr addrspace(1) %arg
@@ -6669,6 +8618,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, 2.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 8
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6698,6 +8687,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, 2.0
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 7
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6726,6 +8755,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 7
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 2.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -7082,6 +9151,75 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s37, 0
+; GFX90A-GISEL-NEXT: s_mov_b32 s36, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s38, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s39, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s40, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s41, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s42, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s43, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s44, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s45, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s46, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s47, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s48, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s49, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s50, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s51, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s52, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s53, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s54, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s55, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s56, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s57, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s58, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s59, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s60, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s61, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s62, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s63, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s64, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s65, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s66, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s67, s37
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -7149,6 +9287,74 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s37, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s36, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s38, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s39, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s40, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s41, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s42, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s43, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s44, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s45, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s46, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s47, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s48, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s49, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s50, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s51, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s52, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s53, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s54, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s55, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s56, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s57, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s58, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s59, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s60, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s61, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s62, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s63, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s64, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s65, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s66, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s67, s37
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -7215,6 +9421,74 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s37, 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s36, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s38, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s39, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s40, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s41, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s42, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s43, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s44, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s45, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s46, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s47, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s48, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s49, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s50, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s51, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s52, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s53, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s54, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s55, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s56, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s57, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s58, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s59, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s60, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s61, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s62, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s63, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s64, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s65, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s66, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s67, s37
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 1.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
store <32 x float> %mai.1, ptr addrspace(1) %arg
@@ -7313,6 +9587,26 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v6, v[0:3]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7331,6 +9625,26 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7348,6 +9662,26 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
@@ -7448,6 +9782,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7465,6 +9818,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7481,6 +9853,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
@@ -7806,6 +10197,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -7836,6 +10257,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -7865,6 +10316,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-VGPR-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
index 63466f89e668f..b6561d20d49d9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8.xf32(<2 x float>, <2 x float>, <4 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x4.xf32(<2 x float>, <2 x float>, <16 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
index 077529c57a3ef..33bbf557097a3 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
@@ -1,5 +1,5 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16>, <2 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16>, <2 x i16>, <16 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
index 263298d1cd1c8..429294c873ac5 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 85b076f636187..75414d3373b8c 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -3,7 +3,7 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -early-live-intervals < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY942 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
; This is better with 90a
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
index 02e08ee7fd579..e030611308ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
@@ -1,5 +1,5 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefix=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefix=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefix=GCN %s
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64, i64, <4 x i32>, i32, i32, i32)
declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64, i64, <16 x i32>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
index 1c7e2e91d51ae..3af97a8a5ab7e 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
@@ -1,7 +1,7 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32)
More information about the llvm-commits
mailing list