[llvm] AMDGPU/GlobalISel: RegBankLegalize rules for gfx90a/gfx942 MFMAs (PR #194076)
via llvm-commits
llvm-commits at lists.llvm.org
Wed May 13 10:49:04 PDT 2026
https://github.com/vangthao95 updated https://github.com/llvm/llvm-project/pull/194076
>From a664206b9bf274a78047ea4f682e80ae9881d7a7 Mon Sep 17 00:00:00 2001
From: Vang Thao <Vang.Thao at amd.com>
Date: Fri, 24 Apr 2026 17:30:58 -0400
Subject: [PATCH 1/2] AMDGPU/GlobalISel: RegBankLegalize rules for
gfx90a/gfx942 MFMAs
Add rules for gfx90a/gfx942 MFMA/SMFMAC intrinsics.
I see some regressions with imm splat tests and stores that could
have taken agprs. I will try to address those in a follow-up patch.
---
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 68 +-
.../GlobalISel/llvm.amdgcn.mfma.gfx90a.ll | 2 +-
.../regbankselect-amdgcn.mfma.gfx90a.mir | 185 +-
.../regbankselect-amdgcn.mfma.gfx942.mir | 281 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 1657 ++++++++++-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll | 2 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll | 308 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 2487 ++++++++++++++++-
.../AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll | 2 +-
.../AMDGPU/mfma-bf16-vgpr-cd-select.ll | 2 +-
llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll | 4 +-
.../AMDGPU/mfma-no-register-aliasing.ll | 2 +-
.../AMDGPU/mfma-vgpr-cd-select-gfx942.ll | 2 +-
.../CodeGen/AMDGPU/mfma-vgpr-cd-select.ll | 4 +-
14 files changed, 4590 insertions(+), 416 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 77802f5f4349a..9da044343862c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -16,6 +16,7 @@
#include "AMDGPURegBankLegalizeRules.h"
#include "AMDGPUInstrInfo.h"
#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/MachineUniformityAnalysis.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -2031,21 +2032,64 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr32}})
.Div(S16, {{Vgpr16}, {IntrId, Vgpr32}});
- // TODO: Add handling for GFX90A+ which should use VGPRs instead of AGPRs.
+ // Predicates to check if we want to use the VGPR form or the AGPR form
+ // for an MFMA / SMFMAC intrinsic on gfx90a+.
+ Predicate needAGPR([](const MachineInstr &MI) -> bool {
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ unsigned MinNumRegsRequired = DstSize / 32;
+ const SIMachineFunctionInfo *Info =
+ MI.getMF()->getInfo<SIMachineFunctionInfo>();
+ return Info->selectAGPRFormMFMA(MinNumRegsRequired);
+ });
+ Predicate needVGPR = !needAGPR;
+
bool HasGFX90AInsts = ST->hasGFX90AInsts();
- addRulesForIOpcs({amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
- amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
- amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
- amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
- amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
- amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
- amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
- amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
- amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
- amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
+ addRulesForIOpcs(
+ {amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
+ amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
+ amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
+ amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
+ amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
+ amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
+ amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
+ amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
+ amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
+ amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16,
+ amdgcn_mfma_f32_32x32x4bf16_1k, amdgcn_mfma_f32_16x16x4bf16_1k,
+ amdgcn_mfma_f32_4x4x4bf16_1k, amdgcn_mfma_f32_32x32x8bf16_1k,
+ amdgcn_mfma_f32_16x16x16bf16_1k, amdgcn_mfma_f64_16x16x4f64,
+ amdgcn_mfma_f64_4x4x4f64, amdgcn_mfma_i32_16x16x32_i8,
+ amdgcn_mfma_i32_32x32x16_i8, amdgcn_mfma_f32_16x16x8_xf32,
+ amdgcn_mfma_f32_32x32x4_xf32, amdgcn_mfma_f32_16x16x32_bf8_bf8,
+ amdgcn_mfma_f32_16x16x32_bf8_fp8, amdgcn_mfma_f32_16x16x32_fp8_bf8,
+ amdgcn_mfma_f32_16x16x32_fp8_fp8, amdgcn_mfma_f32_32x32x16_bf8_bf8,
+ amdgcn_mfma_f32_32x32x16_bf8_fp8, amdgcn_mfma_f32_32x32x16_fp8_bf8,
+ amdgcn_mfma_f32_32x32x16_fp8_fp8})
.Any({{DivAnyTy},
{{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
- !HasGFX90AInsts);
+ !HasGFX90AInsts)
+ .Any({{{DivAnyTy}, needVGPR},
+ {{VgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, VgprAnyTy}}},
+ HasGFX90AInsts)
+ .Any({{{DivAnyTy}, needAGPR},
+ {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
+ HasGFX90AInsts);
+
+ addRulesForIOpcs(
+ {amdgcn_smfmac_f32_16x16x32_f16, amdgcn_smfmac_f32_32x32x16_f16,
+ amdgcn_smfmac_f32_16x16x32_bf16, amdgcn_smfmac_f32_32x32x16_bf16,
+ amdgcn_smfmac_i32_16x16x64_i8, amdgcn_smfmac_i32_32x32x32_i8,
+ amdgcn_smfmac_f32_16x16x64_bf8_bf8, amdgcn_smfmac_f32_16x16x64_bf8_fp8,
+ amdgcn_smfmac_f32_16x16x64_fp8_bf8, amdgcn_smfmac_f32_16x16x64_fp8_fp8,
+ amdgcn_smfmac_f32_32x32x32_bf8_bf8, amdgcn_smfmac_f32_32x32x32_bf8_fp8,
+ amdgcn_smfmac_f32_32x32x32_fp8_bf8, amdgcn_smfmac_f32_32x32x32_fp8_fp8})
+ .Any(
+ {{{DivAnyTy}, needVGPR},
+ {{VgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, VgprAnyTy, VgprAnyTy}}})
+ .Any({{{DivAnyTy}, needAGPR},
+ {{AgprAnyTy},
+ {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy, VgprAnyTy}}});
// WMMA/SWMMAC intrinsics: all register operands map to VGPR.
addRulesForIOpcs(
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
index 5c45565b4d047..86bcac99a22a7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mfma.gfx90a.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN %s
+; RUN: llc -verify-machineinstrs -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
index 5f040779dd42d..ed24ab123584d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx90a.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect -regbankselect-fast -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=regbankselect -regbankselect-greedy -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass='amdgpu-regbankselect,amdgpu-regbanklegalize' -amdgpu-mfma-vgpr-form=0 %s -o - | FileCheck %s
---
name: mfma_f32_32x32x4bf16_1k_vva
@@ -10,22 +9,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-LABEL: name: mfma_f32_32x32x4bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_32x32x4bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_32x32x4bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -41,22 +34,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-LABEL: name: mfma_f32_16x16x4bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_16x16x4bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_16x16x4bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -72,22 +59,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_f32_4x4x4bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_4x4x4bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_4x4x4bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -103,22 +84,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-LABEL: name: mfma_f32_32x32x8bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_32x32x8bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<32 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_32x32x8bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<32 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
@@ -134,22 +109,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_f32_16x16x16bf16_1k_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_16x16x16bf16_1k_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_16x16x16bf16_1k_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16bf16.1k), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -165,22 +134,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; FAST-LABEL: name: mfma_f64_16x16x4f64_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<8 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.16x16x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<8 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INT]](<8 x s32>)
- ; GREEDY-LABEL: name: mfma_f64_16x16x4f64_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<8 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.16x16x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<8 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INT]](<8 x s32>)
+ ; CHECK-LABEL: name: mfma_f64_16x16x4f64_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<8 x s32>) = COPY [[COPY2]](<8 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<8 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.16x16x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<8 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<8 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<8 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[COPY4]](<8 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<8 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
@@ -196,22 +159,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
- ; FAST-LABEL: name: mfma_f64_4x4x4f64_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<2 x s32>) = COPY $agpr0_agpr1
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<2 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.4x4x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<2 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1 = COPY [[INT]](<2 x s32>)
- ; GREEDY-LABEL: name: mfma_f64_4x4x4f64_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<2 x s32>) = COPY $agpr0_agpr1
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<2 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.4x4x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<2 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1 = COPY [[INT]](<2 x s32>)
+ ; CHECK-LABEL: name: mfma_f64_4x4x4f64_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<2 x s32>) = COPY $agpr0_agpr1
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<2 x s32>) = COPY [[COPY2]](<2 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<2 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f64.4x4x4f64), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<2 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<2 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY4]](<2 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<2 x s32>) = COPY $agpr0_agpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
index 12208c133a306..120ec7912496a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.gfx942.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=regbankselect -regbankselect-fast -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=FAST
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=regbankselect -regbankselect-greedy -amdgpu-mfma-vgpr-form=0 -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=GREEDY
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass='amdgpu-regbankselect,amdgpu-regbanklegalize' -amdgpu-mfma-vgpr-form=0 %s -o - | FileCheck %s
---
name: mfma_i32_16x16x32_i8_vva
@@ -10,22 +9,16 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_i32_16x16x32_i8_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x32.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_i32_16x16x32_i8_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x32.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_i32_16x16x32_i8_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x32.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -41,22 +34,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-LABEL: name: mfma_i32_32x32x16_i8_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x16.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: mfma_i32_32x32x16_i8_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x16.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: mfma_i32_32x32x16_i8_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x16.i8), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -72,22 +59,16 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-LABEL: name: mfma_f32_16x16x8_xf32_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_16x16x8_xf32_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<4 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_16x16x8_xf32_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<4 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -103,22 +84,16 @@ body: |
bb.0:
liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-LABEL: name: mfma_f32_32x32x4_xf32_vva
- ; FAST: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: mfma_f32_32x32x4_xf32_vva
- ; GREEDY: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY2]](<16 x s32>), 0, 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: mfma_f32_32x32x4_xf32_vva
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4.xf32), [[COPY]](s64), [[COPY1]](s64), [[COPY3]](<16 x s32>), 0, 0, 0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -134,24 +109,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_16x16x32_f16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_16x16x32_f16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_16x16x32_f16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<4 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY5]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -168,24 +136,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_32x32x16_f16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_32x32x16_f16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_32x32x16_f16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.f16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<16 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY5]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -202,24 +163,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_16x16x32_bf16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_16x16x32_bf16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_16x16x32_bf16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.16x16x32.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<4 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY5]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -236,24 +190,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-LABEL: name: smfmac_f32_32x32x16_bf16_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: smfmac_f32_32x32x16_bf16_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: smfmac_f32_32x32x16_bf16_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.f32.32x32x16.bf16), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<16 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY5]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
@@ -270,24 +217,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-LABEL: name: smfmac_i32_16x16x64_i8_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.16x16x64.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
- ; GREEDY-LABEL: name: smfmac_i32_16x16x64_i8_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.16x16x64.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<4 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>)
+ ; CHECK-LABEL: name: smfmac_i32_16x16x64_i8_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.16x16x64.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<4 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY5]](<4 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3
@@ -304,24 +244,17 @@ body: |
bb.0:
liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-LABEL: name: smfmac_i32_32x32x32_i8_vva
- ; FAST: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; FAST-NEXT: {{ $}}
- ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; FAST-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; FAST-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.32x32x32.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; FAST-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
- ; GREEDY-LABEL: name: smfmac_i32_32x32x32_i8_vva
- ; GREEDY: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
- ; GREEDY-NEXT: {{ $}}
- ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
- ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
- ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
- ; GREEDY-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.32x32x32.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY2]](<16 x s32>), [[COPY3]](s32), 0, 0
- ; GREEDY-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>)
+ ; CHECK-LABEL: name: smfmac_i32_32x32x32_i8_vva
+ ; CHECK: liveins: $vgpr1_vgpr2, $vgpr2_vgpr3_vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr20
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>)
+ ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.smfmac.i32.32x32x32.i8), [[COPY]](s64), [[COPY1]](s128), [[COPY4]](<16 x s32>), [[COPY3]](s32), 0, 0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>)
+ ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY5]](<16 x s32>)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
%2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
index e6379bd4dbb15..835d11642f009 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
@@ -1,8 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX90A-VGPR %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX942-VGPR %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-VGPR %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-VGPR-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-VGPR %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-VGPR-GISEL %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32)
@@ -71,6 +75,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x4bf16_1k v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -128,6 +174,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
+; GFX942-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[36:37]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -185,6 +273,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4bf16_1k v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -241,6 +371,48 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[36:37], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s36, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -278,6 +450,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -305,6 +505,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -332,6 +560,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 9
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -358,6 +614,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -386,6 +670,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x4bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -404,6 +707,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -422,6 +744,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 3
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -439,6 +780,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -477,6 +837,35 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -504,6 +893,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -532,6 +949,35 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8bf16_1k v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -558,6 +1004,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[18:19], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_bf16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -586,6 +1060,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x16bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -604,6 +1097,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 5
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -622,6 +1134,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16bf16_1k v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 9
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -639,6 +1170,25 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg)
; GFX942-VGPR-NEXT: s_nop 6
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16bf16_1k:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s4, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16_bf16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 5
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%a = bitcast i64 1 to <4 x i16>
@@ -665,6 +1215,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX90A-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-GISEL-NEXT: s_nop 7
+; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_4x4x4f64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -681,6 +1247,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-GISEL-NEXT: s_nop 7
+; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_4x4x4f64:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -697,6 +1279,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX90A-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 3
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 7
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_4x4x4f64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -712,6 +1310,22 @@ define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double
; GFX942-VGPR-NEXT: s_nop 7
; GFX942-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_4x4x4f64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 7
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double 0.0, i32 0, i32 0, i32 0)
%mai.2 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double %mai.1, i32 1, i32 2, i32 3)
@@ -743,6 +1357,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
@@ -766,6 +1402,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
@@ -789,6 +1447,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
@@ -811,6 +1491,28 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, doubl
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x double>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %in.1, i32 1, i32 2, i32 3)
@@ -836,6 +1538,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -853,6 +1572,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -870,6 +1606,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -886,6 +1639,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> zeroinitializer, i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -911,6 +1681,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -928,6 +1715,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -945,6 +1749,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -961,6 +1782,23 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrs
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 -1 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -986,6 +1824,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1003,6 +1866,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1020,6 +1908,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1036,6 +1949,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double 1.0), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1061,6 +1999,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1078,6 +2041,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1095,6 +2083,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1111,6 +2124,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], -1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double -1.0), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1136,6 +2174,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1153,6 +2216,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1170,6 +2258,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1186,6 +2299,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspa
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 64 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1222,6 +2360,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX90A-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1250,6 +2414,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1278,6 +2468,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1305,6 +2521,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bit
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877906944 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1338,6 +2580,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1363,6 +2631,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1388,6 +2682,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1412,6 +2732,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877907008 to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1445,6 +2791,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX90A-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1470,6 +2842,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX942-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1495,6 +2893,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1519,6 +2943,32 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (<2 x float> splat (float 1.0) to double)), i32 0, i32 0, i32 0)
%mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
@@ -1555,6 +3005,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX90A-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1583,6 +3057,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1611,6 +3109,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1638,6 +3160,30 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, d
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 0.0, double 0.0, double 0.0, double 1.0>, i32 0, i32 0, i32 0)
store <4 x double> %mai.1, ptr addrspace(1) %arg
@@ -1673,6 +3219,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX90A-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1701,6 +3272,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX90A-VGPR: ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1729,6 +3325,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT: s_endpgm
;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 0
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -1756,6 +3377,31 @@ define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 0x405ec000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 123.0, double 123.0, double 123.0, double 123.0>, i32 0, i32 0, i32 0)
store <4 x double> %mai.1, ptr addrspace(1) %arg
@@ -1763,6 +3409,3 @@ bb:
}
attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GCN: {{.*}}
-; VGPR: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
index 700fc6db8549f..f161e84512f25 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-SDAG,GFX942-VGPRCD-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-SDAG,GFX950-VGPRCD-SDAG %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-GISEL,GFX950-VGPRCD-GISEL %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
index 4048f446371ef..f380634090348 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll
@@ -3,8 +3,10 @@
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GFX90A-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR-SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GFX90A-VGPR-GISEL %s
declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32, i32, <16 x i32>, i32, i32, i32)
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32, i32, <4 x i32>, i32, i32, i32)
@@ -145,67 +147,145 @@ define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 {
; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX908-GISEL-NEXT: s_endpgm
;
-; GFX90A-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A: ; %bb.0: ; %bb
-; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v1, 2
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4
-; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5
-; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6
-; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7
-; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8
-; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9
-; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10
-; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11
-; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12
-; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13
-; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14
-; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15
-; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-NEXT: s_nop 15
-; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
-; GFX90A-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
-; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
-; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
-; GFX90A-NEXT: s_endpgm
+; GFX90A-SDAG-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-SDAG: ; %bb.0: ; %bb
+; GFX90A-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a4, s4
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a5, s5
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a6, s6
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a7, s7
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a8, s8
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a9, s9
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a10, s10
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a11, s11
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a12, s12
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a13, s13
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a14, s14
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a15, s15
+; GFX90A-SDAG-NEXT: s_nop 1
+; GFX90A-SDAG-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-SDAG-NEXT: s_nop 15
+; GFX90A-SDAG-NEXT: s_nop 1
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17]
+; GFX90A-SDAG-NEXT: s_endpgm
;
-; GFX90A-VGPR-LABEL: test_mfma_i32_32x32x8i8:
-; GFX90A-VGPR: ; %bb.0: ; %bb
-; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 2
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 0
-; GFX90A-VGPR-NEXT: s_nop 15
-; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
-; GFX90A-VGPR-NEXT: s_endpgm
+; GFX90A-GISEL-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a4, s4
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a5, s5
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a6, s6
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a7, s7
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a8, s8
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a9, s9
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a10, s10
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a11, s11
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a12, s12
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a13, s13
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a14, s14
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a15, s15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GFX90A-GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
+; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-VGPR-SDAG: ; %bb.0: ; %bb
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: s_nop 1
+; GFX90A-VGPR-SDAG-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-SDAG-NEXT: s_nop 15
+; GFX90A-VGPR-SDAG-NEXT: s_nop 1
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-SDAG-NEXT: s_endpgm
+;
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_i32_32x32x8i8:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_i32_32x32x8i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 15
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -242,41 +322,77 @@ define amdgpu_kernel void @test_mfma_i32_16x16x16i8(ptr addrspace(1) %arg) #0 {
; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GFX908-NEXT: s_endpgm
;
-; GFX90A-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A: ; %bb.0: ; %bb
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-NEXT: v_mov_b32_e32 v0, 1
-; GFX90A-NEXT: v_mov_b32_e32 v2, 2
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0
-; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1
-; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2
-; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3
-; GFX90A-NEXT: s_nop 1
-; GFX90A-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-NEXT: s_nop 10
-; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
-; GFX90A-NEXT: s_endpgm
+; GFX90A-SDAG-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-SDAG: ; %bb.0: ; %bb
+; GFX90A-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v2, 2
+; GFX90A-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-SDAG-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-SDAG-NEXT: s_nop 1
+; GFX90A-SDAG-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v2, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-SDAG-NEXT: s_nop 10
+; GFX90A-SDAG-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
+; GFX90A-SDAG-NEXT: s_endpgm
+;
+; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v1, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a0, s0
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a1, s1
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a2, s2
+; GFX90A-GISEL-NEXT: v_accvgpr_write_b32 a3, s3
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
+; GFX90A-VGPR-SDAG-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-VGPR-SDAG: ; %bb.0: ; %bb
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v6, 2
+; GFX90A-VGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-SDAG-NEXT: s_nop 1
+; GFX90A-VGPR-SDAG-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-SDAG-NEXT: s_nop 10
+; GFX90A-VGPR-SDAG-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-VGPR-SDAG-NEXT: s_endpgm
;
-; GFX90A-VGPR-LABEL: test_mfma_i32_16x16x16i8:
-; GFX90A-VGPR: ; %bb.0: ; %bb
-; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 1
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 2
-; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90A-VGPR-NEXT: s_nop 1
-; GFX90A-VGPR-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v6, v[0:3] cbsz:1 abid:2 blgp:3
-; GFX90A-VGPR-NEXT: s_nop 10
-; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
-; GFX90A-VGPR-NEXT: s_endpgm
+; GFX90A-VGPR-GISEL-LABEL: test_mfma_i32_16x16x16i8:
+; GFX90A-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-VGPR-GISEL-NEXT: s_nop 1
+; GFX90A-VGPR-GISEL-NEXT: v_mfma_i32_16x16x16i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-VGPR-GISEL-NEXT: s_nop 9
+; GFX90A-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 8152b5e38477c..5eeefe90296a0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -3,9 +3,12 @@
; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-NOLIT-SRCC %s
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s
; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-LIT-SRCC %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GFX90A-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GFX942-GISEL %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR-GISEL %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32)
@@ -469,6 +472,46 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -525,6 +568,46 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -580,6 +663,46 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 1, i32 2, i32 3)
@@ -814,6 +937,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -840,6 +989,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -865,6 +1040,32 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3)
@@ -974,6 +1175,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -991,6 +1209,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1007,6 +1242,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1243,6 +1495,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x2f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x2f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1270,6 +1549,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x2f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -1296,6 +1602,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x2f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x2_f32 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float 1.0, float 2.0, <16 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1405,6 +1738,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x4f32:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1422,6 +1772,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f32:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -1438,6 +1805,23 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 9
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4f32:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_f32 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x4f32(float 1.0, float 2.0, <4 x float> %in.1, i32 1, i32 2, i32 3)
@@ -1907,6 +2291,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[32:33], s[40:41], s[40:41] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[34:35], s[42:43], s[42:43] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x4f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -1967,6 +2392,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[40:41]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[42:43]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
@@ -2026,6 +2492,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x4f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[40:41]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[42:43]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x4_2b_f16 v[0:31], v[32:33], v[34:35], v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2275,6 +2782,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x4f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2304,6 +2838,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2332,6 +2893,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x4f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x4_4b_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2456,6 +3044,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x4f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x4f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2476,6 +3082,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2495,6 +3119,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x4f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x4_16b_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2746,6 +3388,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x8f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2775,6 +3445,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
@@ -2803,6 +3500,33 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -2927,6 +3651,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x16f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x16f16:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2947,6 +3689,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 5
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16f16:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
@@ -2966,6 +3726,24 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr
; GFX942-VGPR-NEXT: s_nop 6
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x16f16:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 5
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%c.1 = load <4 x half>, ptr addrspace(1) %c
@@ -3395,6 +4173,46 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_32x32x4i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_32x32x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -3451,6 +4269,46 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_32x32x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -3506,6 +4364,46 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_32x32x4i8:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_32x32x4_2b_i8 v[0:31], v32, v33, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 1, i32 2, <32 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -3740,6 +4638,32 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_16x16x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -3766,6 +4690,32 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 9
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -3791,6 +4741,32 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 9
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -3971,6 +4947,46 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 64
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_i32_16x16x4i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 8
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 1
@@ -3987,6 +5003,46 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 64
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1
@@ -4002,6 +5058,46 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 64
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_16x16x4_4b_i8 v[0:15], v16, v17, v[0:15] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 1, i32 2, <16 x i32> splat (i32 64), i32 1, i32 2, i32 3)
@@ -4111,6 +5207,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX90A-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_4x4x4i8:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -4128,6 +5241,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 3
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -4144,6 +5274,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 {
; GFX942-VGPR-NEXT: s_nop 4
; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 3
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> %in.1, i32 1, i32 2, i32 3)
@@ -4244,6 +5391,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1
@@ -4257,6 +5423,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1
@@ -4269,6 +5454,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace(
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 1), i32 1, i32 2, i32 3)
@@ -4367,6 +5571,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX90A-GISEL: ; %bb.0:
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX90A-GISEL-NEXT: s_movk_i32 s0, 0x41
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_i32_4x4x4i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; GFX942: ; %bb.0:
; GFX942-NEXT: v_mov_b32_e32 v5, 1
@@ -4384,6 +5607,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-GISEL-NEXT: s_movk_i32 s0, 0x41
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
; GFX942-VGPR: ; %bb.0:
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1
@@ -4400,6 +5642,25 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1)
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1:
+; GFX942-VGPR-GISEL: ; %bb.0:
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GFX942-VGPR-GISEL-NEXT: s_movk_i32 s0, 0x41
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_i32_4x4x4_16b_i8 v[0:3], v4, v5, v[0:3] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
%in.1 = load <4 x i32>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> splat (i32 65), i32 1, i32 2, i32 3)
store <4 x i32> %mai.1, ptr addrspace(1) %arg
@@ -4857,6 +6118,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[24:25], s[24:25] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[26:27], s[26:27] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[28:29], s[28:29] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[30:31], s[30:31] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -4914,6 +6216,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
@@ -4970,6 +6313,47 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <32 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
@@ -5207,6 +6591,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_nop 9
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -5234,6 +6645,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
@@ -5260,6 +6698,33 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1)
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <16 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> %in.1, i32 0, i32 0, i32 0)
@@ -5374,6 +6839,24 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -5393,6 +6876,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
@@ -5411,6 +6913,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) %
; GFX942-VGPR-NEXT: s_nop 3
; GFX942-VGPR-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%in.1 = load <4 x float>, ptr addrspace(1) %arg
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> %in.1, i32 0, i32 0, i32 0)
@@ -5512,6 +7033,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -5525,6 +7065,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -5537,6 +7096,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <4 x float> %mai.1, ptr addrspace(1) %arg
@@ -5726,6 +7304,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 8
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 1.0
@@ -5742,6 +7360,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 7
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 1.0
@@ -5757,6 +7415,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 7
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -5968,6 +7666,51 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, 0x40004000
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s2
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x8f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -5986,6 +7729,50 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s2, 0x40004000
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s2
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 8
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 0x3c003c00
@@ -6003,6 +7790,50 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x8f16_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, 0x40004000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s2
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x8_f16 v[0:15], v[16:17], v[18:19], v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 8
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>, <4 x half> <half 2.0, half 2.0, half 2.0, half 2.0>, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -6322,6 +8153,27 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX90A-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, 0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v32, 1.0
@@ -6342,6 +8194,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX942-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 1.0
@@ -6361,6 +8233,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) %
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
store <32 x float> %mai.1, ptr addrspace(1) %arg
@@ -6449,6 +8341,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, 2.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6464,6 +8375,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s1, 2.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6478,6 +8408,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float 1.0, float 2.0, <4 x float> <float 1.0, float 2.0, float 1.0, float 1.0>, i32 0, i32 0, i32 0)
store <4 x float> %mai.1, ptr addrspace(1) %arg
@@ -6669,6 +8618,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s23, 2.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX90A-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[14:15], s[14:15] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[18:19], s[18:19] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[20:21], s[20:21] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[22:23], s[22:23] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_16x16x1f32 v[0:15], v16, v17, v[0:15]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 8
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_16x16x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6698,6 +8687,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s23, 2.0
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 7
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -6726,6 +8755,46 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s8, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s23, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s12, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s13, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s14, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s15, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s16, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s17, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s18, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s19, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s20, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s21, s8
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s22, s8
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v17, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_16x16x1_4b_f32 v[0:15], v16, v17, v[0:15]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 7
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 1.0, float 2.0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 2.0>, i32 0, i32 0, i32 0)
store <16 x float> %mai.1, ptr addrspace(1) %arg
@@ -7082,6 +9151,75 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX90A-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s37, 0
+; GFX90A-GISEL-NEXT: s_mov_b32 s36, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s38, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s39, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s40, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s41, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s42, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s43, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s44, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s45, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s46, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s47, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s48, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s49, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s50, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s51, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s52, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s53, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s54, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s55, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s56, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s57, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s58, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s59, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s60, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s61, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s62, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s63, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s64, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s65, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s66, s37
+; GFX90A-GISEL-NEXT: s_mov_b32 s67, s37
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v32, v33, v[0:31]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 0
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_imm:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
@@ -7149,6 +9287,74 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s37, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s36, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s38, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s39, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s40, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s41, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s42, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s43, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s44, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s45, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s46, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s47, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s48, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s49, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s50, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s51, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s52, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s53, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s54, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s55, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s56, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s57, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s58, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s59, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s60, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s61, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s62, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s63, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s64, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s65, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s66, s37
+; GFX942-GISEL-NEXT: s_mov_b32 s67, s37
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_imm:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0
@@ -7215,6 +9421,74 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT: global_store_dwordx4 v1, v[2:5], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s37, 0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s36, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s38, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s39, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s40, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s41, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s42, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s43, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s44, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s45, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s46, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s47, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s48, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s49, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s50, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s51, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s52, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s53, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s54, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s55, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s56, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s57, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s58, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s59, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s60, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s61, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s62, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s63, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s64, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s65, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s66, s37
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s67, s37
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_nop 0
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 1.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
store <32 x float> %mai.1, ptr addrspace(1) %arg
@@ -7313,6 +9587,26 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v6, v[0:3]
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 3
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7331,6 +9625,26 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 2
+; GFX942-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7348,6 +9662,26 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_lshlrev_b32_e32 v5, 4, v0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v6, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v6, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 2
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
@@ -7448,6 +9782,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX90A-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX90A-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX90A-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX90A-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: s_nop 1
+; GFX90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v4, v5, v[0:3]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7465,6 +9818,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 1.0
@@ -7481,6 +9853,25 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa
; GFX942-VGPR-NEXT: s_nop 2
; GFX942-VGPR-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s0, 0x42f60000
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s1, s0
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s2, s0
+; GFX942-VGPR-GISEL-NEXT: s_mov_b32 s3, s0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_4x4x1_16b_f32 v[0:3], v4, v5, v[0:3]
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
@@ -7806,6 +10197,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX90A-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX90A-NEXT: s_endpgm
;
+; GFX90A-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX90A-GISEL: ; %bb.0: ; %bb
+; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-GISEL-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX90A-GISEL-NEXT: s_nop 15
+; GFX90A-GISEL-NEXT: s_nop 2
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX90A-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX90A-GISEL-NEXT: s_endpgm
+;
; GFX942-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -7836,6 +10257,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-NEXT: s_endpgm
;
+; GFX942-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX942-GISEL: ; %bb.0: ; %bb
+; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-GISEL-NEXT: s_nop 15
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-GISEL-NEXT: s_endpgm
+;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; GFX942-VGPR: ; %bb.0: ; %bb
; GFX942-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -7865,6 +10316,36 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
; GFX942-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT: s_endpgm
+;
+; GFX942-VGPR-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
+; GFX942-VGPR-GISEL: ; %bb.0: ; %bb
+; GFX942-VGPR-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-VGPR-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-VGPR-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GFX942-VGPR-GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-VGPR-GISEL-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31] cbsz:1 abid:2 blgp:3
+; GFX942-VGPR-GISEL-NEXT: s_nop 15
+; GFX942-VGPR-GISEL-NEXT: s_nop 1
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-VGPR-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-VGPR-GISEL-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %arg, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
index 63466f89e668f..b6561d20d49d9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.xf32.gfx942.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8.xf32(<2 x float>, <2 x float>, <4 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x4.xf32(<2 x float>, <2 x float>, <16 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
index 077529c57a3ef..33bbf557097a3 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-bf16-vgpr-cd-select.ll
@@ -1,5 +1,5 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16>, <2 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16>, <2 x i16>, <16 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
index 263298d1cd1c8..429294c873ac5 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX90A %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 85b076f636187..75414d3373b8c 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -3,7 +3,7 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -early-live-intervals < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY942 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
; This is better with 90a
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
index 02e08ee7fd579..e030611308ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select-gfx942.ll
@@ -1,5 +1,5 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefix=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefix=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefix=GCN %s
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64, i64, <4 x i32>, i32, i32, i32)
declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64, i64, <16 x i32>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
index 1c7e2e91d51ae..3af97a8a5ab7e 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-vgpr-cd-select.ll
@@ -1,7 +1,7 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --enable-var-scope --check-prefixes=GCN %s
declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32, i32, i32)
>From 09c25df5e4d763cc2af25c711b5bde2606cea08e Mon Sep 17 00:00:00 2001
From: Vang Thao <Vang.Thao at amd.com>
Date: Wed, 13 May 2026 13:24:48 -0400
Subject: [PATCH 2/2] Split gfx90a+ only mfmas. Remove needVGPR.
---
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 55 +++++++++++--------
1 file changed, 31 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 9da044343862c..39ebfdab38ffd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -2042,40 +2042,47 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
MI.getMF()->getInfo<SIMachineFunctionInfo>();
return Info->selectAGPRFormMFMA(MinNumRegsRequired);
});
- Predicate needVGPR = !needAGPR;
bool HasGFX90AInsts = ST->hasGFX90AInsts();
- addRulesForIOpcs(
- {amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
- amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
- amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
- amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
- amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
- amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
- amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
- amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
- amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
- amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16,
- amdgcn_mfma_f32_32x32x4bf16_1k, amdgcn_mfma_f32_16x16x4bf16_1k,
- amdgcn_mfma_f32_4x4x4bf16_1k, amdgcn_mfma_f32_32x32x8bf16_1k,
- amdgcn_mfma_f32_16x16x16bf16_1k, amdgcn_mfma_f64_16x16x4f64,
- amdgcn_mfma_f64_4x4x4f64, amdgcn_mfma_i32_16x16x32_i8,
- amdgcn_mfma_i32_32x32x16_i8, amdgcn_mfma_f32_16x16x8_xf32,
- amdgcn_mfma_f32_32x32x4_xf32, amdgcn_mfma_f32_16x16x32_bf8_bf8,
- amdgcn_mfma_f32_16x16x32_bf8_fp8, amdgcn_mfma_f32_16x16x32_fp8_bf8,
- amdgcn_mfma_f32_16x16x32_fp8_fp8, amdgcn_mfma_f32_32x32x16_bf8_bf8,
- amdgcn_mfma_f32_32x32x16_bf8_fp8, amdgcn_mfma_f32_32x32x16_fp8_bf8,
- amdgcn_mfma_f32_32x32x16_fp8_fp8})
+
+ // On gfx90a+ both AGPR-form and VGPR-form exists
+ addRulesForIOpcs({amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
+ amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
+ amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
+ amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
+ amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
+ amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
+ amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
+ amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
+ amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
+ amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
.Any({{DivAnyTy},
{{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
!HasGFX90AInsts)
- .Any({{{DivAnyTy}, needVGPR},
+ .Any({{{DivAnyTy}, !needAGPR},
{{VgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, VgprAnyTy}}},
HasGFX90AInsts)
.Any({{{DivAnyTy}, needAGPR},
{{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
HasGFX90AInsts);
+ // gfx90a+ only MFMAs
+ addRulesForIOpcs(
+ {amdgcn_mfma_f32_32x32x4bf16_1k, amdgcn_mfma_f32_16x16x4bf16_1k,
+ amdgcn_mfma_f32_4x4x4bf16_1k, amdgcn_mfma_f32_32x32x8bf16_1k,
+ amdgcn_mfma_f32_16x16x16bf16_1k, amdgcn_mfma_f64_16x16x4f64,
+ amdgcn_mfma_f64_4x4x4f64, amdgcn_mfma_i32_16x16x32_i8,
+ amdgcn_mfma_i32_32x32x16_i8, amdgcn_mfma_f32_16x16x8_xf32,
+ amdgcn_mfma_f32_32x32x4_xf32, amdgcn_mfma_f32_16x16x32_bf8_bf8,
+ amdgcn_mfma_f32_16x16x32_bf8_fp8, amdgcn_mfma_f32_16x16x32_fp8_bf8,
+ amdgcn_mfma_f32_16x16x32_fp8_fp8, amdgcn_mfma_f32_32x32x16_bf8_bf8,
+ amdgcn_mfma_f32_32x32x16_bf8_fp8, amdgcn_mfma_f32_32x32x16_fp8_bf8,
+ amdgcn_mfma_f32_32x32x16_fp8_fp8})
+ .Any({{{DivAnyTy}, !needAGPR},
+ {{VgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, VgprAnyTy}}})
+ .Any({{{DivAnyTy}, needAGPR},
+ {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}});
+
addRulesForIOpcs(
{amdgcn_smfmac_f32_16x16x32_f16, amdgcn_smfmac_f32_32x32x16_f16,
amdgcn_smfmac_f32_16x16x32_bf16, amdgcn_smfmac_f32_32x32x16_bf16,
@@ -2085,7 +2092,7 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
amdgcn_smfmac_f32_32x32x32_bf8_bf8, amdgcn_smfmac_f32_32x32x32_bf8_fp8,
amdgcn_smfmac_f32_32x32x32_fp8_bf8, amdgcn_smfmac_f32_32x32x32_fp8_fp8})
.Any(
- {{{DivAnyTy}, needVGPR},
+ {{{DivAnyTy}, !needAGPR},
{{VgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, VgprAnyTy, VgprAnyTy}}})
.Any({{{DivAnyTy}, needAGPR},
{{AgprAnyTy},
More information about the llvm-commits
mailing list