[llvm] [AMDGPU][NFC] Move isXDL and isDGEMM to SIInstrInfo. (PR #129103)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 27 11:19:36 PST 2025
https://github.com/sstipano updated https://github.com/llvm/llvm-project/pull/129103
>From 04733e9e9a3b7af47a45950a4d459281d6e9ee42 Mon Sep 17 00:00:00 2001
From: Stefan Stipanovic <sstipano7 at gmail.com>
Date: Thu, 27 Feb 2025 20:22:23 +0100
Subject: [PATCH] [AMDGPU][NFC] Move isXDL and isDGEMM to SIInstrInfo
---
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 59 +++++++------------
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 14 +++++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 ++
3 files changed, 39 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 1ff75095b220a..582da42a0dc4e 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -111,25 +111,6 @@ static bool isSMovRel(unsigned Opcode) {
}
}
-static bool isDGEMM(unsigned Opcode) {
- return AMDGPU::getMAIIsDGEMM(Opcode);
-}
-
-static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
- unsigned Opcode = MI.getOpcode();
-
- if (!SIInstrInfo::isMAI(MI) ||
- isDGEMM(Opcode) ||
- Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
- Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
- return false;
-
- if (!ST.hasGFX940Insts())
- return true;
-
- return AMDGPU::getMAIIsGFX940XDL(Opcode);
-}
-
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
const MachineInstr &MI) {
if (TII.isAlwaysGDS(MI.getOpcode()))
@@ -2375,7 +2356,8 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
unsigned Opc1 = MI1->getOpcode();
int NeedWaitStates = 0;
if (OpNo == SrcCIdx) {
- if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
+ if (!SIInstrInfo::isDGEMM(Opc) &&
+ (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
NeedWaitStates = 0;
} else if (FullReg) {
if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
@@ -2392,7 +2374,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
- if (!isXDL(ST, *MI))
+ if (!TII.isXDL(*MI))
NeedWaitStates =
ST.hasGFX950Insts()
? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
@@ -2400,18 +2382,18 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
break;
case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
- if (!isXDL(ST, *MI))
+ if (!TII.isXDL(*MI))
NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
break;
default:
int NumPasses = TSchedModel.computeInstrLatency(MI1);
if (ST.hasGFX940Insts()) {
- if (isXDL(ST, *MI) && !isXDL(ST, *MI1))
+ if (TII.isXDL(*MI) && !TII.isXDL(*MI1))
break;
NeedWaitStates =
- isXDL(ST, *MI1)
- ? (isXDL(ST, *MI)
+ TII.isXDL(*MI1)
+ ? (TII.isXDL(*MI)
? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
@@ -2424,18 +2406,19 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
switch (NumPasses) {
case 2:
NeedWaitStates =
- isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
- : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
+ SIInstrInfo::isDGEMM(Opc)
+ ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
+ : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
break;
case 8:
NeedWaitStates =
- isDGEMM(Opc)
+ SIInstrInfo::isDGEMM(Opc)
? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
: SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
break;
case 16:
NeedWaitStates =
- isDGEMM(Opc)
+ SIInstrInfo::isDGEMM(Opc)
? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
: SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
break;
@@ -2464,7 +2447,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
if (ST.hasGFX940Insts()) {
NeedWaitStates =
- isXDL(ST, *MI1)
+ TII.isXDL(*MI1)
? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
@@ -2631,7 +2614,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
return 0;
auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
- return isDGEMM(MI.getOpcode());
+ return SIInstrInfo::isDGEMM(MI.getOpcode());
};
// This is checked in checkMAIHazards90A()
@@ -2670,7 +2653,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
bool DGEMMAfterVALUWrite = false;
auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
// Found DGEMM on reverse traversal to def.
- if (isDGEMM(MI.getOpcode()))
+ if (SIInstrInfo::isDGEMM(MI.getOpcode()))
DGEMMAfterVALUWrite = true;
// Only hazard if register is defined by a VALU and a DGEMM is found after
@@ -2745,7 +2728,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
int NumPasses = HazardDefLatency;
int NeedWaitStates = MaxWaitStates;
- if (isDGEMM(MFMA->getOpcode())) {
+ if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
switch (HazardDefLatency) {
case 4:
NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
@@ -2765,7 +2748,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
}
} else if (ST.hasGFX940Insts()) {
NeedWaitStates =
- isXDL(ST, *MFMA)
+ TII.isXDL(*MFMA)
? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
@@ -2838,7 +2821,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
int NeedWaitStates = MaxWaitStates;
int NumPasses = TSchedModel.computeInstrLatency(MFMA);
- if (isDGEMM(MFMA->getOpcode())) {
+ if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
switch (NumPasses) {
case 4:
NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
@@ -2852,7 +2835,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
}
} else if (ST.hasGFX940Insts()) {
NeedWaitStates =
- isXDL(ST, *MFMA)
+ TII.isXDL(*MFMA)
? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
@@ -2880,11 +2863,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
}
auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
- if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
+ if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) ||
!MI.readsRegister(Reg, &TRI))
return false;
- if (ST.hasGFX940Insts() && !isXDL(ST, MI))
+ if (ST.hasGFX940Insts() && !TII.isXDL(MI))
return false;
const MachineOperand *SrcC =
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9aec2bef0c18a..0face8108d249 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -10251,3 +10251,17 @@ bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
return TargetInstrInfo::isGlobalMemoryObject(MI);
}
+
+bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
+ unsigned Opcode = MI.getOpcode();
+
+ if (!SIInstrInfo::isMAI(MI) || isDGEMM(Opcode) ||
+ Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
+ Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
+ return false;
+
+ if (!ST.hasGFX940Insts())
+ return true;
+
+ return AMDGPU::getMAIIsGFX940XDL(Opcode);
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 428322a5a2f04..88b1e477f13e4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -847,6 +847,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
}
+ bool isXDL(const MachineInstr &MI) const;
+
+ static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }
+
static bool isLDSDIR(const MachineInstr &MI) {
return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
}
More information about the llvm-commits
mailing list