[llvm] [AMDGPU] NFC: Obviously show isVALU includes LDSDMA instructions (PR #203548)
Jeffrey Byrnes via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 09:28:36 PDT 2026
https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/203548
>From f4ae9c36f1b1991041a203cf558b423b25584943 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 12 Jun 2026 07:15:29 -0700
Subject: [PATCH 1/4] [AMDGPU] NFC: Obviously show isVALU includes LDSDMA
instructions
Change-Id: I3854fe397cafad4484c5af53c739e2117287d2c9
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 2 +-
.../lib/Target/AMDGPU/AMDGPUHazardLatency.cpp | 4 +-
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 4 +-
.../Target/AMDGPU/AMDGPUInsertDelayAlu.cpp | 4 +-
.../Target/AMDGPU/AMDGPUSetWavePriority.cpp | 4 +-
.../Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 2 +-
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 82 +++++++++----------
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 6 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 16 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 20 +++--
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 2 +-
13 files changed, 78 insertions(+), 72 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index bee2093cdc12c..2f8d52efbd064 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -65,7 +65,7 @@ InstructionFlavor llvm::AMDGPU::classifyFlavor(const MachineInstr &MI,
if (SII.isTRANS(MI))
return InstructionFlavor::TRANS;
- if (SII.isVALU(MI))
+ if (SII.isVALU(MI, /*AllowLDSDMA=*/true))
return InstructionFlavor::SingleCycleVALU;
if (SII.isDS(MI))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHazardLatency.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHazardLatency.cpp
index 37f86781659c8..8825b3b8cb938 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHazardLatency.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHazardLatency.cpp
@@ -48,7 +48,7 @@ void HazardLatency::apply(ScheduleDAGInstrs *DAG) {
for (SUnit &SU : DAG->SUnits) {
const MachineInstr *MI = SU.getInstr();
- if (!SIInstrInfo::isVALU(*MI))
+ if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
continue;
if (MI->getOpcode() == AMDGPU::V_READLANE_B32 ||
MI->getOpcode() == AMDGPU::V_READFIRSTLANE_B32)
@@ -59,7 +59,7 @@ void HazardLatency::apply(ScheduleDAGInstrs *DAG) {
// Boost latency on VALU writes to SGPRs used by VALUs.
// Reduce risk of premature VALU pipeline stall on associated reads.
MachineInstr *DestMI = SuccDep.getSUnit()->getInstr();
- if (!SIInstrInfo::isVALU(*DestMI))
+ if (!SIInstrInfo::isVALU(*DestMI, /*AllowLDSDMA=*/true))
continue;
Register Reg = SuccDep.getReg();
if (!TRI.isSGPRReg(MRI, Reg))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 6c95dc76f4f6f..fb28f596dbc01 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2446,12 +2446,12 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
}
else if (((SGMask & SchedGroupMask::ALU) != SchedGroupMask::NONE) &&
- (TII->isVALU(MI) || TII->isMFMAorWMMA(MI) || TII->isSALU(MI) ||
+ (TII->isVALU(MI, /*AllowLDSDMA=*/true) || TII->isMFMAorWMMA(MI) || TII->isSALU(MI) ||
TII->isTRANS(MI)))
Result = !MI.mayLoadOrStore();
else if (((SGMask & SchedGroupMask::VALU) != SchedGroupMask::NONE) &&
- TII->isVALU(MI) && !TII->isMFMAorWMMA(MI) && !TII->isTRANS(MI) &&
+ TII->isVALU(MI, /*AllowLDSDMA=*/true) && !TII->isMFMAorWMMA(MI) && !TII->isTRANS(MI) &&
!TII->isLDSDMA(MI)) {
// Some memory instructions may be marked as VALU (e.g. BUFFER_LOAD_*_LDS).
// For our purposes, these shall not be classified as VALU as this results
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index 35390fc71b5e7..c184edc103aa5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -76,7 +76,7 @@ class AMDGPUInsertDelayAlu {
// WMMA XDL ops are treated the same as TRANS.
if (ST->hasGFX1250Insts() && SII->isXDLWMMA(MI))
return TRANS;
- if (SIInstrInfo::isVALU(MI))
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
return VALU;
if (SIInstrInfo::isSALU(MI))
return SALU;
@@ -410,7 +410,7 @@ class AMDGPUInsertDelayAlu {
}
}
- if (SII->isVALU(MI.getOpcode())) {
+ if (SII->isVALU(MI.getOpcode(), /*AllowLDSDMA=*/true)) {
for (const auto &Op : MI.defs()) {
Register Reg = Op.getReg();
if (AMDGPU::isSGPR(Reg, TRI)) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
index 56aa3f6db83ad..7a1570278812d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
@@ -156,7 +156,7 @@ bool AMDGPUSetWavePriority::run(MachineFunction &MF) {
MaxNumVALUInstsInMiddle =
std::max(MaxNumVALUInstsInMiddle, NumVALUInstsAtEnd);
NumVALUInstsAtEnd = 0;
- } else if (SIInstrInfo::isVALU(MI)) {
+ } else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
if (AtStart)
++MBBInfos[MBB].NumVALUInstsAtStart;
++NumVALUInstsAtEnd;
@@ -189,7 +189,7 @@ bool AMDGPUSetWavePriority::run(MachineFunction &MF) {
// Raise the priority at the beginning of the shader.
MachineBasicBlock::iterator I = Entry.begin(), E = Entry.end();
- while (I != E && !SIInstrInfo::isVALU(*I) && !I->isTerminator())
+ while (I != E && !SIInstrInfo::isVALU(*I, /*AllowLDSDMA=*/true) && !I->isTerminator())
++I;
BuildSetprioMI(Entry, I, HighPriority);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index a04163599aab9..1ab11a87518eb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -278,7 +278,7 @@ class AMDGPUWaitSGPRHazards {
}
// Process only VALUs and SALUs
- bool IsVALU = SIInstrInfo::isVALU(*MI);
+ bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
bool IsSALU = SIInstrInfo::isSALU(*MI);
if (!IsVALU && !IsSALU)
continue;
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index e2c6b25dc4319..261558d85db46 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -177,7 +177,7 @@ static bool isPermlane(const MachineInstr &MI) {
}
static bool isLdsDma(const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) &&
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
(SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI));
}
@@ -218,7 +218,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
return HazardType;
- if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
+ if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) && checkVALUHazards(MI) > 0)
return HazardType;
if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
@@ -230,7 +230,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
return HazardType;
- if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
+ if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) || SIInstrInfo::isVMEM(*MI) ||
SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) &&
checkMAIVALUHazards(MI) > 0)
return HazardType;
@@ -359,7 +359,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
if (SIInstrInfo::isVMEM(*MI))
WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
- if (SIInstrInfo::isVALU(*MI))
+ if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
WaitStates = std::max(WaitStates, checkVALUHazards(MI));
if (SIInstrInfo::isDPP(*MI))
@@ -371,7 +371,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
if (isRWLane(MI->getOpcode()))
WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
- if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
+ if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) || SIInstrInfo::isVMEM(*MI) ||
SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) &&
checkMAIVALUHazards(MI) > 0)
WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
@@ -747,7 +747,7 @@ int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
// SGPR was written by a VALU instruction.
int SmrdSgprWaitStates = 4;
auto IsHazardDefFn = [this](const MachineInstr &MI) {
- return TII.isVALU(MI);
+ return TII.isVALU(MI, /*AllowLDSDMA=*/true);
};
auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
return TII.isSALU(MI);
@@ -792,7 +792,7 @@ int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
// SGPR was written by a VALU Instruction.
const int VmemSgprWaitStates = 5;
auto IsHazardDefFn = [this](const MachineInstr &MI) {
- return TII.isVALU(MI);
+ return TII.isVALU(MI, /*AllowLDSDMA=*/true);
};
for (const MachineOperand &Use : VMEM->uses()) {
if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
@@ -815,7 +815,7 @@ int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
int DppExecWaitStates = 5;
int WaitStatesNeeded = 0;
auto IsHazardDefFn = [TII](const MachineInstr &MI) {
- return TII->isVALU(MI);
+ return TII->isVALU(MI, /*AllowLDSDMA=*/true);
};
for (const MachineOperand &Use : DPP->uses()) {
@@ -844,7 +844,7 @@ int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
// instruction.
const int DivFMasWaitStates = 4;
auto IsHazardDefFn = [TII](const MachineInstr &MI) {
- return TII->isVALU(MI);
+ return TII->isVALU(MI, /*AllowLDSDMA=*/true);
};
int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
DivFMasWaitStates);
@@ -999,7 +999,7 @@ int GCNHazardRecognizer::checkVALUHazardsHelper(
/// none exists.
static const MachineOperand *
getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
- if (!SIInstrInfo::isVALU(MI))
+ if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
return nullptr;
const SIInstrInfo *TII = ST.getInstrInfo();
@@ -1129,7 +1129,7 @@ int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
const MachineRegisterInfo &MRI = MF.getRegInfo();
Register UseReg;
auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
- if (!SIInstrInfo::isVALU(MI))
+ if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
return false;
return MI.modifiesRegister(UseReg, TRI);
};
@@ -1267,7 +1267,7 @@ int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
return 0;
Register LaneSelectReg = LaneSelectOp->getReg();
- auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
+ auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI, /*AllowLDSDMA=*/true); };
const int RWLaneWaitStates = 4;
int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
@@ -1353,7 +1353,7 @@ bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
auto IsExpiredFn = [](const MachineInstr &MI, int) {
unsigned Opc = MI.getOpcode();
- return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && Opc != AMDGPU::V_NOP_e32 &&
Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
};
@@ -1403,7 +1403,7 @@ bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
};
auto IsExpiredFn = [](const MachineInstr &MI, int) {
- return SIInstrInfo::isVALU(MI) ||
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
(MI.getOpcode() == AMDGPU::S_WAITCNT &&
!MI.getOperand(0).getImm()) ||
(MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
@@ -1426,7 +1426,7 @@ bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
return false;
assert(!ST.hasExtendedWaitCounts());
- if (!SIInstrInfo::isVALU(*MI))
+ if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
return false;
AMDGPU::OpName SDSTName;
@@ -1516,7 +1516,7 @@ bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
return false;
assert(!ST.hasExtendedWaitCounts());
- if (!SIInstrInfo::isVALU(*MI))
+ if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
return false;
const SIRegisterInfo *TRI = ST.getRegisterInfo();
@@ -1524,14 +1524,14 @@ bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
return false;
auto IsHazardFn = [TRI](const MachineInstr &I) {
- if (SIInstrInfo::isVALU(I))
+ if (SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true))
return false;
return I.readsRegister(AMDGPU::EXEC, TRI);
};
const SIInstrInfo *TII = ST.getInstrInfo();
auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
- if (SIInstrInfo::isVALU(MI)) {
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
return true;
for (auto MO : MI.implicit_operands())
@@ -1647,7 +1647,7 @@ bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
bool VisitedTrans = false;
auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
- if (!SIInstrInfo::isVALU(I))
+ if (!SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true))
return false;
VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I);
// Cover both WAR and WAW
@@ -1661,7 +1661,7 @@ bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
SIInstrInfo::isEXP(I);
};
auto GetWaitStatesFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) ? 1 : 0;
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
};
DenseSet<const MachineBasicBlock *> Visited;
@@ -1697,7 +1697,7 @@ bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
// TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
// according to the type of VMEM instruction.
auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
- return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) ||
+ return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true) || SIInstrInfo::isEXP(I) ||
(I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
(I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
AMDGPU::DepCtr::decodeFieldVmVsrc(I.getOperand(0).getImm()) == 0) ||
@@ -1725,7 +1725,7 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
return false;
assert(!ST.hasExtendedWaitCounts());
- if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI))
+ if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
return false;
SmallSetVector<Register, 4> SrcVGPRs;
@@ -1791,7 +1791,7 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
// Track registers writes
bool Changed = false;
- if (SIInstrInfo::isVALU(I)) {
+ if (SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true)) {
for (Register Src : SrcVGPRs) {
if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) {
State.DefPos[Src] = State.VALUs;
@@ -1862,7 +1862,7 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
return HazardFound;
};
auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
- if (SIInstrInfo::isVALU(MI))
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
State.VALUs += 1;
};
@@ -1882,7 +1882,7 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
return false;
assert(!ST.hasExtendedWaitCounts());
- if (!SIInstrInfo::isVALU(*MI))
+ if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
return false;
SmallSet<Register, 4> SrcVGPRs;
@@ -1944,7 +1944,7 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
return NoHazardFound;
};
auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
- if (SIInstrInfo::isVALU(MI))
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
State.VALUs += 1;
if (SIInstrInfo::isTRANS(MI))
State.TRANS += 1;
@@ -1965,7 +1965,7 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
if (!ST.hasGFX1250Insts() || // Coexecution disabled.
- !SIInstrInfo::isVALU(*MI) || SIInstrInfo::isTRANS(*MI))
+ !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) || SIInstrInfo::isTRANS(*MI))
return false;
const SIInstrInfo *TII = ST.getInstrInfo();
@@ -1997,7 +1997,7 @@ bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
};
auto IsExpiredFn = [](const MachineInstr &I, int) {
- return SIInstrInfo::isVALU(I);
+ return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true);
};
const int HasVALU = std::numeric_limits<int>::max();
@@ -2050,7 +2050,7 @@ bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
};
auto IsExpiredFn = [](const MachineInstr &I, int) {
- return SIInstrInfo::isVALU(I);
+ return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true);
};
if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
@@ -2063,7 +2063,7 @@ bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
}
static bool isCoexecutableVALUInst(const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isWMMA(MI) &&
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isWMMA(MI) &&
!SIInstrInfo::isSWMMAC(MI) && !SIInstrInfo::isLDSDMA(MI);
}
@@ -2167,7 +2167,7 @@ int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
};
auto GetWaitStatesFn = [](const MachineInstr &I) {
- return SIInstrInfo::isVALU(I) ? 1 : 0;
+ return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true) ? 1 : 0;
};
int WaitStatesNeeded = -1;
@@ -2490,7 +2490,7 @@ int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
};
auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
- if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
+ if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
return true;
return SIInstrInfo::isWaitcnt(MI.getOpcode());
@@ -2541,7 +2541,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
unsigned Opc = MI->getOpcode();
auto IsVALUFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) || MI.isInlineAsm();
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) || MI.isInlineAsm();
};
if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
@@ -2755,11 +2755,11 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
unsigned Opc = MI->getOpcode();
auto IsLegacyVALUFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isMFMA(MI);
};
auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) &&
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isMFMA(MI) &&
!SIInstrInfo::isDOT(MI);
};
@@ -2988,7 +2988,7 @@ int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
return false;
auto IsVALUFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isMAI(MI);
};
return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
std::numeric_limits<int>::max();
@@ -3013,7 +3013,7 @@ int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
};
auto IsVALUFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true);
};
const int VCmpXWritesExecWaitStates = 4;
@@ -3096,7 +3096,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
bool IsMem = SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI);
bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
- bool IsVALU = SIInstrInfo::isVALU(*MI);
+ bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
const MachineInstr *MFMA = nullptr;
unsigned Reg;
@@ -3125,7 +3125,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
// Only hazard if register is defined by a VALU and a DGEMM is found after
// after the def.
- if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite)
+ if (!TII.isVALU(MI, /*AllowLDSDMA=*/true) || !DGEMMAfterVALUWrite)
return false;
return true;
@@ -3439,7 +3439,7 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
return false;
const bool IsSALU = SIInstrInfo::isSALU(*MI);
- const bool IsVALU = SIInstrInfo::isVALU(*MI);
+ const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
if (!IsSALU && !IsVALU)
return false;
@@ -3861,7 +3861,7 @@ bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
// This literally abuses the idea of waitstates. Instead of waitstates it
// returns 1 for SGPR written and 0 otherwise.
auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
- if (!TII->isSALU(MI) && !TII->isVALU(MI))
+ if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/true))
return 0;
for (const MachineOperand &MO : MI.all_defs()) {
if (TRI->isSGPRReg(MRI, MO.getReg()))
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index ffe34102b0a4e..0e5e7a3e069fe 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1008,7 +1008,7 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
}
} else if (Inst->getNumExplicitDefs() != 0) {
Register Reg = Inst->getOperand(0).getReg();
- if (Reg.isVirtual() && TRI->isSGPRReg(*MRI, Reg) && !TII->isVALU(*Inst)) {
+ if (Reg.isVirtual() && TRI->isSGPRReg(*MRI, Reg) && !TII->isVALU(*Inst, /*AllowLDSDMA=*/true)) {
for (auto &U : MRI->use_instructions(Reg))
Users.push_back(&U);
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 30950f8f5d9e5..321225259f36f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5826,7 +5826,7 @@ getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
llvm_unreachable("unhandled lane op");
}
unsigned ClampOpc = Opc;
- if (!ST.getInstrInfo()->isVALU(Opc)) {
+ if (!ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
if (Opc == AMDGPU::S_SUB_I32)
ClampOpc = AMDGPU::S_ADD_I32;
if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 6bccf1eea5047..390024f3eece5 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1683,7 +1683,7 @@ MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
if (!Wait.hasWait())
return Reg;
- if (Context->TII.isVALU(MI))
+ if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/true))
return Reg32;
// If hi/lo16 mixed events
@@ -2778,7 +2778,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// waits on VA_VDST if the instruction it would precede is not a VALU
// instruction, since hardware handles VALU->VGPR->VALU hazards in
// expert scheduling mode.
- if (TII.isVALU(MI))
+ if (TII.isVALU(MI, /*AllowLDSDMA=*/true))
Wait.set(AMDGPU::VA_VDST, ~0u);
// Since the translation for VMEM addresses occur in-order, we can apply the
@@ -2858,7 +2858,7 @@ bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
std::optional<WaitEventType>
SIInsertWaitcnts::getExpertSchedulingEventType(const MachineInstr &Inst) const {
- if (TII.isVALU(Inst)) {
+ if (TII.isVALU(Inst, /*AllowLDSDMA=*/true)) {
// Core/Side-, DP-, XDL- and TRANS-MACC VALU instructions complete
// out-of-order with respect to each other, so each of these classes
// has its own event.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index a5198cce50e0f..40479c194f3a8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -152,7 +152,7 @@ bool SIInstrInfo::isReMaterializableImpl(
// Returns true if the result of a VALU instruction depends on exec.
bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
- assert(isVALU(MI));
+ assert(isVALU(MI, /*AllowLDSDMA=*/true));
// If it is convergent it depends on EXEC.
if (MI.isConvergent())
@@ -175,7 +175,7 @@ bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
// Any implicit use of exec by VALU is not a real register read.
return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
- isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent());
+ isVALU(*MO.getParent(), /*AllowLDSDMA=*/true) && !resultDependsOnExec(*MO.getParent());
}
bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
@@ -2892,7 +2892,7 @@ bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
// It may move literal to position other than src0, this is not allowed
// pre-gfx10 However, most test cases need literals in Src0 for VOP
// FIXME: After gfx9, literal can be in place other than Src0
- if (isVALU(MI)) {
+ if (isVALU(MI, /*AllowLDSDMA=*/true)) {
if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
!isInlineConstant(MO0, OpInfo1))
return false;
@@ -5170,7 +5170,7 @@ static Register findImplicitSGPRRead(const MachineInstr &MI) {
}
static bool shouldReadExec(const MachineInstr &MI) {
- if (SIInstrInfo::isVALU(MI)) {
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
switch (MI.getOpcode()) {
case AMDGPU::V_READLANE_B32:
case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
@@ -5581,7 +5581,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
// Verify VOP*. Ignore multiple sgpr operands on writelane.
- if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
+ if (isVALU(MI, /*AllowLDSDMA=*/true) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
unsigned ConstantBusCount = 0;
bool UsesLiteral = false;
const MachineOperand *LiteralVal = nullptr;
@@ -6571,7 +6571,7 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
const bool IsInlineConst = !MO->isReg() && isInlineConstant(*MO, OpInfo);
- if (isVALU(MI) && !IsInlineConst && usesConstantBus(MRI, *MO, OpInfo)) {
+ if (isVALU(MI, /*AllowLDSDMA=*/true) && !IsInlineConst && usesConstantBus(MRI, *MO, OpInfo)) {
const MachineOperand *UsedLiteral = nullptr;
int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
@@ -6688,7 +6688,7 @@ bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {
if (!IsGFX950Only && !IsGFX940Only)
return false;
- if (!isVALU(MI))
+ if (!isVALU(MI, /*AllowLDSDMA=*/true))
return false;
// V_COS, V_EXP, V_RCP, etc.
@@ -10006,7 +10006,7 @@ unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
// Instructions may have a 32-bit literal encoded after them. Check
// operands that could ever be literals.
- if (isVALU(MI) || isSALU(MI)) {
+ if (isVALU(MI, /*AllowLDSDMA=*/true) || isSALU(MI)) {
if (isDPP(MI))
return DescSize;
bool HasLiteral = false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 228549b7e383f..4beabfdbb7647 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -491,11 +491,17 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return get(Opcode).TSFlags & SIInstrFlags::SALU;
}
- static bool isVALU(const MachineInstr &MI) {
+ static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
+ if (!AllowLDSDMA && isLDSDMA(MI))
+ return false;
+
return MI.getDesc().TSFlags & SIInstrFlags::VALU;
}
- bool isVALU(uint32_t Opcode) const {
+ bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
+ if (!AllowLDSDMA && isLDSDMA(Opcode))
+ return false;
+
return get(Opcode).TSFlags & SIInstrFlags::VALU;
}
@@ -646,12 +652,12 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
static bool isLDSDMA(const MachineInstr &MI) {
- return (isVALU(MI) && (isMUBUF(MI) || isFLAT(MI))) ||
+ return (isVALU(MI, /*AllowLDSDMA=*/true) && (isMUBUF(MI) || isFLAT(MI))) ||
(MI.getDesc().TSFlags & SIInstrFlags::TENSOR_CNT);
}
- bool isLDSDMA(uint32_t Opcode) {
- return (isVALU(Opcode) && (isMUBUF(Opcode) || isFLAT(Opcode))) ||
+ bool isLDSDMA(uint32_t Opcode) const {
+ return (isVALU(Opcode, /*AllowLDSDMA=*/true) && (isMUBUF(Opcode) || isFLAT(Opcode))) ||
(get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT);
}
@@ -893,13 +899,13 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
static bool isVGPRSpill(const MachineInstr &MI) {
return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
- (isSpill(MI) && isVALU(MI));
+ (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/true));
}
bool isVGPRSpill(uint32_t Opcode) const {
return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
- (isSpill(Opcode) && isVALU(Opcode));
+ (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/true));
}
static bool isSGPRSpill(const MachineInstr &MI) {
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 9cc86e84407b1..c04ceae75c53d 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -379,7 +379,7 @@ void SILowerControlFlow::emitIfBreak(MachineInstr &MI) {
if (MI.getOperand(1).isReg()) {
if (MachineInstr *Def = MRI->getUniqueVRegDef(MI.getOperand(1).getReg())) {
SkipAnding = Def->getParent() == MI.getParent()
- && SIInstrInfo::isVALU(*Def);
+ && SIInstrInfo::isVALU(*Def, /*AllowLDSDMA=*/true);
}
}
>From a1811b8841e3d9a2aed3284d70669deee23fada9 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 12 Jun 2026 07:41:26 -0700
Subject: [PATCH 2/4] Formatting
Change-Id: I0fbcad129f96986d2a448bfa4b5a027a2a5c07bd
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 8 ++--
.../Target/AMDGPU/AMDGPUSetWavePriority.cpp | 3 +-
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 43 ++++++++++++-------
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 9 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 3 +-
llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp | 4 +-
7 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index fb28f596dbc01..ee827c36f5b18 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2446,13 +2446,13 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
}
else if (((SGMask & SchedGroupMask::ALU) != SchedGroupMask::NONE) &&
- (TII->isVALU(MI, /*AllowLDSDMA=*/true) || TII->isMFMAorWMMA(MI) || TII->isSALU(MI) ||
- TII->isTRANS(MI)))
+ (TII->isVALU(MI, /*AllowLDSDMA=*/true) || TII->isMFMAorWMMA(MI) ||
+ TII->isSALU(MI) || TII->isTRANS(MI)))
Result = !MI.mayLoadOrStore();
else if (((SGMask & SchedGroupMask::VALU) != SchedGroupMask::NONE) &&
- TII->isVALU(MI, /*AllowLDSDMA=*/true) && !TII->isMFMAorWMMA(MI) && !TII->isTRANS(MI) &&
- !TII->isLDSDMA(MI)) {
+ TII->isVALU(MI, /*AllowLDSDMA=*/true) && !TII->isMFMAorWMMA(MI) &&
+ !TII->isTRANS(MI) && !TII->isLDSDMA(MI)) {
// Some memory instructions may be marked as VALU (e.g. BUFFER_LOAD_*_LDS).
// For our purposes, these shall not be classified as VALU as this results
// in unexpected behavior.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
index 7a1570278812d..e3448be9f44f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
@@ -189,7 +189,8 @@ bool AMDGPUSetWavePriority::run(MachineFunction &MF) {
// Raise the priority at the beginning of the shader.
MachineBasicBlock::iterator I = Entry.begin(), E = Entry.end();
- while (I != E && !SIInstrInfo::isVALU(*I, /*AllowLDSDMA=*/true) && !I->isTerminator())
+ while (I != E && !SIInstrInfo::isVALU(*I, /*AllowLDSDMA=*/true) &&
+ !I->isTerminator())
++I;
BuildSetprioMI(Entry, I, HighPriority);
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 261558d85db46..11094934ef01c 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -218,7 +218,8 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
return HazardType;
- if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) && checkVALUHazards(MI) > 0)
+ if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) &&
+ checkVALUHazards(MI) > 0)
return HazardType;
if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
@@ -230,8 +231,9 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
return HazardType;
- if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) || SIInstrInfo::isVMEM(*MI) ||
- SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) &&
+ if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
+ SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI) ||
+ SIInstrInfo::isEXP(*MI)) &&
checkMAIVALUHazards(MI) > 0)
return HazardType;
@@ -371,8 +373,9 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
if (isRWLane(MI->getOpcode()))
WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
- if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) || SIInstrInfo::isVMEM(*MI) ||
- SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) &&
+ if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
+ SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI) ||
+ SIInstrInfo::isEXP(*MI)) &&
checkMAIVALUHazards(MI) > 0)
WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
@@ -1267,7 +1270,9 @@ int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
return 0;
Register LaneSelectReg = LaneSelectOp->getReg();
- auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI, /*AllowLDSDMA=*/true); };
+ auto IsHazardFn = [TII](const MachineInstr &MI) {
+ return TII->isVALU(MI, /*AllowLDSDMA=*/true);
+ };
const int RWLaneWaitStates = 4;
int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
@@ -1353,8 +1358,9 @@ bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
auto IsExpiredFn = [](const MachineInstr &MI, int) {
unsigned Opc = MI.getOpcode();
- return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && Opc != AMDGPU::V_NOP_e32 &&
- Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
+ Opc != AMDGPU::V_NOP_sdwa;
};
if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
@@ -1697,7 +1703,8 @@ bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
// TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
// according to the type of VMEM instruction.
auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
- return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true) || SIInstrInfo::isEXP(I) ||
+ return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true) ||
+ SIInstrInfo::isEXP(I) ||
(I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
(I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
AMDGPU::DepCtr::decodeFieldVmVsrc(I.getOperand(0).getImm()) == 0) ||
@@ -1965,7 +1972,8 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
if (!ST.hasGFX1250Insts() || // Coexecution disabled.
- !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) || SIInstrInfo::isTRANS(*MI))
+ !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
+ SIInstrInfo::isTRANS(*MI))
return false;
const SIInstrInfo *TII = ST.getInstrInfo();
@@ -2063,8 +2071,9 @@ bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
}
static bool isCoexecutableVALUInst(const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isWMMA(MI) &&
- !SIInstrInfo::isSWMMAC(MI) && !SIInstrInfo::isLDSDMA(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
+ !SIInstrInfo::isLDSDMA(MI);
}
// Classify XDL WMMA instructions into co-execution hazard categories
@@ -2755,12 +2764,13 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
unsigned Opc = MI->getOpcode();
auto IsLegacyVALUFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isMFMA(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ !SIInstrInfo::isMFMA(MI);
};
auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isMFMA(MI) &&
- !SIInstrInfo::isDOT(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
};
if (!SIInstrInfo::isMFMA(*MI))
@@ -2988,7 +2998,8 @@ int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
return false;
auto IsVALUFn = [](const MachineInstr &MI) {
- return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isMAI(MI);
+ return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ !SIInstrInfo::isMAI(MI);
};
return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
std::numeric_limits<int>::max();
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 0e5e7a3e069fe..9a3bc28134824 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1008,7 +1008,8 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
}
} else if (Inst->getNumExplicitDefs() != 0) {
Register Reg = Inst->getOperand(0).getReg();
- if (Reg.isVirtual() && TRI->isSGPRReg(*MRI, Reg) && !TII->isVALU(*Inst, /*AllowLDSDMA=*/true)) {
+ if (Reg.isVirtual() && TRI->isSGPRReg(*MRI, Reg) &&
+ !TII->isVALU(*Inst, /*AllowLDSDMA=*/true)) {
for (auto &U : MRI->use_instructions(Reg))
Users.push_back(&U);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 40479c194f3a8..f8eae46afa8d6 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -175,7 +175,8 @@ bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
// Any implicit use of exec by VALU is not a real register read.
return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
- isVALU(*MO.getParent(), /*AllowLDSDMA=*/true) && !resultDependsOnExec(*MO.getParent());
+ isVALU(*MO.getParent(), /*AllowLDSDMA=*/true) &&
+ !resultDependsOnExec(*MO.getParent());
}
bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
@@ -5581,7 +5582,8 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
// Verify VOP*. Ignore multiple sgpr operands on writelane.
- if (isVALU(MI, /*AllowLDSDMA=*/true) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
+ if (isVALU(MI, /*AllowLDSDMA=*/true) &&
+ Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
unsigned ConstantBusCount = 0;
bool UsesLiteral = false;
const MachineOperand *LiteralVal = nullptr;
@@ -6571,7 +6573,8 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
const bool IsInlineConst = !MO->isReg() && isInlineConstant(*MO, OpInfo);
- if (isVALU(MI, /*AllowLDSDMA=*/true) && !IsInlineConst && usesConstantBus(MRI, *MO, OpInfo)) {
+ if (isVALU(MI, /*AllowLDSDMA=*/true) && !IsInlineConst &&
+ usesConstantBus(MRI, *MO, OpInfo)) {
const MachineOperand *UsedLiteral = nullptr;
int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 4beabfdbb7647..b327c208f5753 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -657,7 +657,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
bool isLDSDMA(uint32_t Opcode) const {
- return (isVALU(Opcode, /*AllowLDSDMA=*/true) && (isMUBUF(Opcode) || isFLAT(Opcode))) ||
+ return (isVALU(Opcode, /*AllowLDSDMA=*/true) &&
+ (isMUBUF(Opcode) || isFLAT(Opcode))) ||
(get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT);
}
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index c04ceae75c53d..8a4f978ddc1a0 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -378,8 +378,8 @@ void SILowerControlFlow::emitIfBreak(MachineInstr &MI) {
bool SkipAnding = false;
if (MI.getOperand(1).isReg()) {
if (MachineInstr *Def = MRI->getUniqueVRegDef(MI.getOperand(1).getReg())) {
- SkipAnding = Def->getParent() == MI.getParent()
- && SIInstrInfo::isVALU(*Def, /*AllowLDSDMA=*/true);
+ SkipAnding = Def->getParent() == MI.getParent() &&
+ SIInstrInfo::isVALU(*Def, /*AllowLDSDMA=*/true);
}
}
>From 664a595f33003784629c1aca4f4e5daa990b282f Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 12 Jun 2026 08:06:26 -0700
Subject: [PATCH 3/4] Rebase for new isVALU calls
Change-Id: Id2280498a63994268e902d90b787e32fdccc912a
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 321225259f36f..e310ca5654990 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6199,7 +6199,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
LaneValueReg)
.addReg(SrcReg)
.addReg(FF1Reg);
- if (ST.getInstrInfo()->isVALU(Opc)) {
+ if (ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
// Get the Lane Value in VGPR to avoid the Constant Bus Restriction
Register LaneValVgpr = MRI.createVirtualRegister(SrcRegClass);
Register VgprResultReg = MRI.createVirtualRegister(SrcRegClass);
@@ -6221,7 +6221,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
OpInstr.addImm(0); // opsel
if (hasOMod)
OpInstr.addImm(0); // omod
- if (ST.getInstrInfo()->isVALU(Opc)) {
+ if (ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
DstReg)
.addReg(OpDstReg);
>From c51ba1cb2a9bc8de3c2eda8d386489a930457ce1 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 12 Jun 2026 09:27:23 -0700
Subject: [PATCH 4/4] inline isVALU into isLDSDMA to avoid recursive calls
Change-Id: I5b7e0c09fd310639ebf918d0152a419dca415798
---
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index b327c208f5753..7b140897e5d30 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -652,12 +652,13 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
static bool isLDSDMA(const MachineInstr &MI) {
- return (isVALU(MI, /*AllowLDSDMA=*/true) && (isMUBUF(MI) || isFLAT(MI))) ||
+ return ((MI.getDesc().TSFlags & SIInstrFlags::VALU) &&
+ (isMUBUF(MI) || isFLAT(MI))) ||
(MI.getDesc().TSFlags & SIInstrFlags::TENSOR_CNT);
}
bool isLDSDMA(uint32_t Opcode) const {
- return (isVALU(Opcode, /*AllowLDSDMA=*/true) &&
+ return ((get(Opcode).TSFlags & SIInstrFlags::VALU) &&
(isMUBUF(Opcode) || isFLAT(Opcode))) ||
(get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT);
}
More information about the llvm-commits
mailing list