[llvm-branch-commits] [llvm] [AMDGPU] Add and plug-in DSLatencyMode flag + attr to control LDS latency (PR #205626)
Jeffrey Byrnes via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 8 12:47:12 PDT 2026
https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/205626
>From 3c9c5644c2102a908b691f276552e82ec8df0041 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 15 Sep 2026 20:12:22 -0700
Subject: [PATCH 1/5] [AMDGPU] Add DSLatencyMode flag + attr to control LDS
latency
Change-Id: Ia5fee7983adffdb837ba2e876a64d805263b60c3
---
.../Target/AMDGPU/AMDGPUBarrierLatency.cpp | 5 +-
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 1 +
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 144 +++++++++++++-----
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 3 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.cpp | 9 ++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 61 +++++++-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 19 +++
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 4 +-
.../coexec-sched-flavor-classification.mir | 2 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 4 +
.../CodeGen/AMDGPU/ds-latency-mode-attr.mir | 61 ++++++++
.../AMDGPU/ds-latency-mode-branch-cost.mir | 53 +++++++
.../ds-latency-mode-default-scheduler.mir | 119 +++++++++++++++
.../CodeGen/AMDGPU/ds-latency-mode-flag.mir | 29 ++++
14 files changed, 467 insertions(+), 47 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp b/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp
index 02bad827d51b03..789d0261be00d0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp
@@ -101,7 +101,6 @@ void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {
const unsigned BarrierSignalWaitLatency = BarrierSignalWaitLatencyOpt;
SmallVector<SUnit *, 8> RegionTDM;
SmallVector<SUnit *, 8> RegionAsync;
- const TargetSchedModel *SchedModel = DAG->getSchedModel();
for (SUnit &SU : DAG->SUnits) {
const MachineInstr *MI = SU.getInstr();
@@ -124,9 +123,9 @@ void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {
if (!MI->mayLoad() || MI->mayStore())
continue;
+ unsigned InstrLatency = TII->getInstrLatency(*MI);
addLatencyToEdge(PredDep, SU,
- SchedModel ? SchedModel->computeInstrLatency(MI, false)
- : FenceLatency);
+ InstrLatency ? InstrLatency : FenceLatency);
}
} else if (Op == AMDGPU::S_BARRIER_WAIT) {
for (SDep &PredDep : SU.Preds) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 482df5f0408d35..dce9f4ccbed9b4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -1343,6 +1343,7 @@ ScheduleDAGInstrs *
llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
<< C->MF->getName() << '\n');
+
ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
C, std::make_unique<AMDGPUCoExecSchedStrategy>(C));
DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index b100c14f7b5dbe..894e632f129128 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -2765,7 +2765,7 @@ static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
unsigned Category = 0;
- unsigned Latency = SchedModel.computeInstrLatency(&MI);
+ unsigned Latency = TII->getInstrLatency(MI);
switch (Latency) {
case 4:
// Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
@@ -3225,8 +3225,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
Register DstReg = MI.getOperand(0).getReg();
if (DstReg == Reg)
return false;
- HazardDefLatency =
- std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
+ HazardDefLatency = std::max(HazardDefLatency, TII.getInstrLatency(MI));
return TRI.regsOverlap(DstReg, Reg);
};
@@ -3302,8 +3301,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
if (!SIInstrInfo::isMFMA(MI))
return false;
Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
- HazardDefLatency =
- std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
+ HazardDefLatency = std::max(HazardDefLatency, TII.getInstrLatency(MI));
return TRI.regsOverlap(Reg, DstReg);
};
@@ -3586,15 +3584,81 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
int OpNo = Use.getOperandNo();
- WaitStatesNeeded = std::max(
- WaitStatesNeeded,
- getMaxWindowDeficit(
- MaxWaitStates, [&](const MachineInstr &P) -> std::optional<int> {
- if (!SIInstrInfo::isMFMA(P) ||
- !TRI.regsOverlap(P.getOperand(0).getReg(), Reg))
- return std::nullopt;
- return getMFMAReadWaitStates(*MI, P, Reg, OpNo == SrcCIdx);
- }));
+ unsigned Opc1 = MI1->getOpcode();
+ int NeedWaitStates = 0;
+ if (OpNo == SrcCIdx) {
+ if (!SIInstrInfo::isDGEMM(Opc) &&
+ (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
+ NeedWaitStates = 0;
+ } else if (FullReg) {
+ if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
+ Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
+ (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
+ Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
+ NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
+ else if (ST.hasGFX940Insts() && TII.getInstrLatency(*MI1) == 2)
+ NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
+
+ // The accumulator forwarding path that allows zero wait states is only
+ // available while the chain stays on a single MFMA. Two different MFMAs
+ // sharing an accumulator need the wait states of a partial overlap.
+ if (ST.hasGFX940Insts() && !isSameMFMA(Opc, Opc1)) {
+ NeedWaitStates = std::max(NeedWaitStates,
+ getMFMAOverlappedSrcCWaitStates(MI, MI1));
+ }
+ } else {
+ NeedWaitStates = getMFMAOverlappedSrcCWaitStates(MI, MI1);
+ }
+ } else {
+ switch (Opc1) {
+ case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
+ case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
+ case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
+ case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
+ NeedWaitStates =
+ ST.hasGFX950Insts()
+ ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
+ : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
+ break;
+ case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
+ case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
+ NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
+ break;
+ default:
+ int NumPasses = TII.getInstrLatency(*MI1);
+
+ if (ST.hasGFX940Insts()) {
+ NeedWaitStates =
+ TII.isXDL(*MI1)
+ ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
+ NumPasses, ST.hasGFX950Insts())
+ : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
+ NumPasses);
+ break;
+ }
+
+ switch (NumPasses) {
+ case 2:
+ NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
+ break;
+ case 4:
+ llvm_unreachable("unexpected number of passes for mfma");
+ case 8:
+ NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
+ break;
+ case 16:
+ default:
+ NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
+ }
+ }
+ }
+ assert(NeedWaitStates <= MaxWaitStates &&
+ "hazard requirement exceeds the scan window");
+ if (WaitStatesNeeded >= NeedWaitStates)
+ continue;
+
+ WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
+ WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
if (WaitStatesNeeded == MaxWaitStates)
break;
@@ -3841,26 +3905,18 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
int NumPasses = TSchedModel.computeInstrLatency(&Producer);
int NeedWaitStates = MaxWaitStates;
- if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
- switch (NumPasses) {
- case 4:
- NeedWaitStates = IsMemOrExport
- ? DMFMA4x4WriteVgprMemExpReadWaitStates
- : DMFMA4x4WriteVgprVALUReadWaitStates;
- break;
- case 8:
- case 16:
- NeedWaitStates =
- IsMemOrExport
- ? DMFMA16x16WriteVgprMemExpReadWaitStates
- : (ST.hasGFX950Insts()
- ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
- : DMFMA16x16WriteVgprVALUReadWaitStates);
- break;
- default:
- llvm_unreachable("unexpected dgemm");
- }
- } else if (ST.hasGFX940Insts()) {
+ unsigned HazardDefLatency = TII.getInstrLatency(*MFMA);
+ int NumPasses = HazardDefLatency;
+ int NeedWaitStates = MaxWaitStates;
+
+ if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
+ switch (HazardDefLatency) {
+ case 4:
+ NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
+ : DMFMA4x4WriteVgprVALUReadWaitStates;
+ break;
+ case 8:
+ case 16:
NeedWaitStates =
TII.isXDL(Producer)
? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
@@ -3942,7 +3998,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
auto MFMAWawWindow = [&](const MachineInstr &Producer) {
int NeedWaitStates = MaxWaitStates;
- int NumPasses = TSchedModel.computeInstrLatency(&Producer);
+ int NumPasses = TII.getInstrLatency(*MFMA);
if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
switch (NumPasses) {
@@ -3992,8 +4048,20 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
return MFMAWawWindow(P);
}));
- if (WaitStatesNeeded >= MaxWaitStates)
- break;
+ unsigned HazardDefLatency = TII.getInstrLatency(*MFMA);
+ int NeedWaitStates = MaxWaitStates;
+ switch (HazardDefLatency) {
+ case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
+ break;
+ case 4: assert(ST.hasGFX940Insts());
+ NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
+ break;
+ case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
+ break;
+ case 16: [[fallthrough]];
+ default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
+ break;
+ }
// Wait states this write needs after an MFMA that read Reg as srcC.
auto SMFMAReadAsCWindow =
@@ -4052,7 +4120,7 @@ bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
if (IsMFMAFn(*MI)) {
int W = getWaitStatesSince(IsMFMAFn, 16);
if (MAI)
- return W < (int)TSchedModel.computeInstrLatency(MAI);
+ return W < (int)TII.getInstrLatency(*MAI);
}
return false;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 584e06b9e069e0..7f19a7056dcfda 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2052,10 +2052,11 @@ GCNSchedStage::computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle,
DenseMap<unsigned, unsigned> &ReadyCycles,
const TargetSchedModel &SM) {
unsigned ReadyCycle = CurrCycle;
+ const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG.TII);
for (auto &D : SU.Preds) {
if (D.isAssignedRegDep()) {
MachineInstr *DefMI = D.getSUnit()->getInstr();
- unsigned Latency = SM.computeInstrLatency(DefMI);
+ unsigned Latency = SII->getInstrLatency(*DefMI);
unsigned DefReady = ReadyCycles[DAG.getSUnit(DefMI)->NodeNum];
ReadyCycle = std::max(ReadyCycle, DefReady + Latency);
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 199c594c4083bd..2ad02f053eb5ae 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -852,6 +852,15 @@ void GCNSubtarget::adjustSchedDependency(
return; // This is not a data dependency anymore.
}
+ // DS load/store latency is variable depending on LDS contention.
+ if (InstrInfo.isDS(*DefI) &&
+ InstrInfo.getDSLatencyMultiplier(*DefI->getMF()) != 1) {
+ // For LDS instructions, we have overrides to change default latencies.
+ unsigned Latency = InstrInfo.getInstrLatency(*DefI);
+ Dep.setLatency(Latency);
+ return;
+ }
+
if (DefI->isBundle()) {
const SIRegisterInfo *TRI = getRegisterInfo();
auto Reg = Dep.getReg();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0050a26313d8de..787a7326f13257 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
+#include "AMDGPUTargetMachine.h"
#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
@@ -70,6 +71,17 @@ static cl::opt<bool> Fix16BitCopies(
cl::init(true),
cl::ReallyHidden);
+static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency(
+ "amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"),
+ cl::values(
+ clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast",
+ "Use default/pinned latency (no contention)"),
+ clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded",
+ "Use loaded latency (moderate contention, 3x latency)"),
+ clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded",
+ "Use overloaded latency (high contention, 5x latency)")),
+ cl::init(SIInstrInfo::DSLatencyMode::Fast), cl::Hidden);
+
SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
: AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
AMDGPU::ADJCALLSTACKDOWN),
@@ -11232,12 +11244,24 @@ unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
unsigned Lat = 0, Count = 0;
for (++I; I != E && I->isBundledWithPred(); ++I) {
++Count;
- Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I));
+ Lat = std::max(Lat, getInstrLatency(*I));
}
return Lat + Count - 1;
}
- return SchedModel.computeInstrLatency(&MI);
+ return getInstrLatency(MI);
+}
+
+unsigned SIInstrInfo::getInstrLatency(const MachineInstr &MI) const {
+ if (SchedModel.hasInstrSchedModel()) {
+ unsigned Latency = SchedModel.computeInstrLatency(&MI);
+ if (isDS(MI)) {
+ Latency *= getDSLatencyMultiplier(*MI.getMF());
+ }
+ return Latency;
+ }
+
+ return 0;
}
unsigned SIInstrInfo::getBlockingCycles(const MachineInstr &MI) const {
@@ -11982,3 +12006,36 @@ bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
return AMDGPU::getMAIIsGFX940XDL(Opcode);
}
+
+unsigned SIInstrInfo::getDSLatencyMultiplier(const MachineFunction &MF) {
+ const Function &F = MF.getFunction();
+
+ // Priority selection goes to the attribute
+ Attribute A = F.getFnAttribute("amdgpu-ds-latency-mode");
+ if (A.isValid()) {
+ StringRef Val = A.getValueAsString();
+ if (Val == "fast")
+ return 1;
+ if (Val == "loaded")
+ return 3;
+ if (Val == "overloaded")
+ return 5;
+ }
+
+ // If using coexec scheduler, default to "loaded" mode unless overridden
+ // by the command line option.
+ if (DSLatency.getNumOccurrences() == 0 &&
+ AMDGPU::getSchedStrategy(F) == "coexec")
+ return 3;
+
+ switch (DSLatency) {
+ case DSLatencyMode::Fast:
+ return 1; // Use default scheduling model latency
+ case DSLatencyMode::Loaded:
+ return 3;
+ case DSLatencyMode::Overloaded:
+ return 5;
+ }
+
+ return 1;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index f589d0299442bc..595fff7e029a41 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1261,6 +1261,21 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
}
+ /// DS latency modes. The latency of DS load/store instructions
+ /// is variable depending on LDS contention.
+ enum class DSLatencyMode {
+ Fast, ///< Use default/pinned latency (no contention)
+ Loaded, ///< Use loaded latency (moderate contention, 3x latency)
+ Overloaded ///< Use overloaded latency (high contention, 5x latency)
+ };
+
+ /// \p returns the DS instruction latency multiplier based on the selected
+ /// DSLatencyMode. \p returns 1 if the default
+ /// scheduling model latency should be used (fast mode).
+ /// Checks the function attribute first, then if using coexec scheduler
+ /// defaults to "loaded", then falls back to the global command line option.
+ static unsigned getDSLatencyMultiplier(const MachineFunction &MF);
+
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
switch (Opcode) {
case AMDGPU::S_WAITCNT_soft:
@@ -1801,11 +1816,15 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
LiveIntervals *LIS = nullptr,
VirtRegMap *VRM = nullptr) const override;
+ // Silence a hidden overloaded virtual function warning.
+ using TargetInstrInfo::getInstrLatency;
+
unsigned getInstrLatency(const InstrItineraryData *ItinData,
const MachineInstr &MI,
unsigned *PredCost = nullptr) const override;
unsigned getBlockingCycles(const MachineInstr &MI) const;
+ unsigned getInstrLatency(const MachineInstr &MI) const;
/// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
/// Returns 0 if \p MI is not in the table. Used as a multi-pass VALU denylist
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 89461de967b85d..53f063923d403d 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -415,14 +415,14 @@ class BranchWeightCostModel {
BranchProb = Head.getSuccProbability(FromIt);
if (BranchProb.isUnknown())
BranchProb = BranchProbability::getZero();
- BranchTakenCost = SchedModel.computeInstrLatency(&Branch);
+ BranchTakenCost = TII.getInstrLatency(Branch);
}
bool isProfitable(const MachineInstr &MI) {
if (TII.isWaitcnt(MI.getOpcode()))
return false;
- ThenCyclesCost += SchedModel.computeInstrLatency(&MI);
+ ThenCyclesCost += TII.getInstrLatency(MI);
// Consider `P = N/D` to be the probability of execz being false (skipping
// the then-block) The transformation is profitable if always executing the
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
index 2a1a53ab88ffcd..7fcfa684a514e5 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
@@ -7,7 +7,7 @@
# CHECK-DAG: TRANS: 4 cycles, 2 instrs
# CHECK-DAG: VALU(Nc): 36 cycles, 2 instrs
# CHECK-DAG: VMEM: 3200 cycles, 10 instrs
-# CHECK-DAG: DS: 5 cycles, 4 instrs
+# CHECK-DAG: DS: 15 cycles, 4 instrs
# CHECK-DAG: DMA: 640 cycles, 2 instrs
# GFX1251: HWUI Resource Pressure:
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 88cb777095229a..c6efd5b0ca0bed 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -321,6 +321,10 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
+; COEXEC-NEXT: s_wait_dscnt 0x1c
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; COEXEC-NEXT: s_wait_dscnt 0x18
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x14
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x10
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
new file mode 100644
index 00000000000000..59bc0c0554c9c9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
@@ -0,0 +1,61 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler %s -filetype=null 2>&1 | FileCheck %s
+
+
+# CHECK: Region: test_ds_latency_fast
+# CHECK: DS: 80 cycles, 4 instrs
+
+# CHECK: Region: test_ds_latency_loaded
+# CHECK: DS: 240 cycles, 4 instrs
+
+# CHECK: Region: test_ds_latency_overloaded
+# CHECK: DS: 400 cycles, 4 instrs
+
+--- |
+ define void @test_ds_latency_fast() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="fast" { ret void }
+ define void @test_ds_latency_loaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="loaded" { ret void }
+ define void @test_ds_latency_overloaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="overloaded" { ret void }
+
+...
+
+---
+name: test_ds_latency_fast
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
+
+---
+name: test_ds_latency_loaded
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
+
+---
+name: test_ds_latency_overloaded
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
new file mode 100644
index 00000000000000..95bf9df77965a8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
@@ -0,0 +1,53 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s
+
+# Test that ds-latency-mode affects the branch-over-removal cost model in
+# SIPreEmitPeephole. With higher DS latency, the cost of the then-block
+# is higher, making branch removal less profitable.
+
+---
+name: skip_execz_ds_multi
+body: |
+ ; FAST-LABEL: name: skip_execz_ds_multi
+ ; FAST: bb.0:
+ ; FAST-NEXT: successors: %bb.1(0x78000000)
+ ; FAST-NEXT: {{ $}}
+ ; FAST-NEXT: bb.1:
+ ; FAST-NEXT: successors: %bb.2(0x80000000)
+ ; FAST-NEXT: {{ $}}
+ ; FAST-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; FAST-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; FAST-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec
+ ; FAST-NEXT: {{ $}}
+ ; FAST-NEXT: bb.2:
+ ; FAST-NEXT: S_ENDPGM 0
+ ;
+ ; OVERLOADED-LABEL: name: skip_execz_ds_multi
+ ; OVERLOADED: bb.0:
+ ; OVERLOADED-NEXT: successors: %bb.1(0x78000000), %bb.2(0x08000000)
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: bb.1:
+ ; OVERLOADED-NEXT: successors: %bb.2(0x80000000)
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; OVERLOADED-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; OVERLOADED-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: bb.2:
+ ; OVERLOADED-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x78000000), %bb.2(0x08000000)
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ bb.1:
+ successors: %bb.2
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
new file mode 100644
index 00000000000000..41695b863b00a5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
@@ -0,0 +1,119 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s
+
+---
+name: test_ds_latency_default_sched
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; FAST-LABEL: name: test_ds_latency_default_sched
+ ; FAST: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; FAST-NEXT: $m0 = S_MOV_B32 -1
+ ; FAST-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[DEF]], 0, 0, implicit $m0, implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DS_READ_B32_]], [[DS_READ_B32_]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_24:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_28:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_29:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_4]]
+ ;
+ ; OVERLOADED-LABEL: name: test_ds_latency_default_sched
+ ; OVERLOADED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; OVERLOADED-NEXT: $m0 = S_MOV_B32 -1
+ ; OVERLOADED-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[DEF]], 0, 0, implicit $m0, implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: [[V_ADD_U32_e32_24:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DS_READ_B32_]], [[DS_READ_B32_]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_28:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_29:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_24]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ $m0 = S_MOV_B32 -1
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+ %3:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %4:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %5:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %6:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %15:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %16:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %17:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %18:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %19:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %20:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %21:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %22:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %23:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %24:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %25:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %26:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %27:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %28:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %29:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %30:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %31:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %32:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ S_ENDPGM 0, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
new file mode 100644
index 00000000000000..527390abcc8f58
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
@@ -0,0 +1,29 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=fast %s -filetype=null 2>&1 | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=loaded %s -filetype=null 2>&1 | FileCheck -check-prefix=LOADED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -filetype=null 2>&1 | FileCheck -check-prefix=OVERLOADED %s
+
+# Test that the -amdgpu-ds-latency-mode command line flag affects DS cycle
+# accounting in the coexec scheduler.
+# DS instructions have base latency of 20 cycles each (4 instructions = 80 cycles).
+# - fast: 4 * 20 * 1 = 80 cycles
+# - loaded: 4 * 20 * 3 = 240 cycles
+# - overloaded: 4 * 20 * 5 = 400 cycles
+
+# FAST: DS: 80 cycles, 4 instrs
+# LOADED: DS: 240 cycles, 4 instrs
+# OVERLOADED: DS: 400 cycles, 4 instrs
+
+---
+name: test_ds_latency_flag
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
>From d86e1cdf53355a10cbae69916ea457881ab31263 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 7 Jul 2026 18:03:38 -0700
Subject: [PATCH 2/5] More documentation
Change-Id: I65ff1a1b605796d21a52766dcf04c64bc64aaa4e
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 27 ++++++++++++++++++++------
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 787a7326f13257..3da21a26efc980 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -75,7 +75,7 @@ static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency(
"amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"),
cl::values(
clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast",
- "Use default/pinned latency (no contention)"),
+ "Use default/pinged latency (no contention)"),
clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded",
"Use loaded latency (moderate contention, 3x latency)"),
clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded",
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 595fff7e029a41..5bd5f39981b8f6 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1261,17 +1261,29 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
}
- /// DS latency modes. The latency of DS load/store instructions
- /// is variable depending on LDS contention.
+ /// DS latency modes. The latency of an individual DS load/store instruction
+ /// is variable. Some of the major sources that cause this variation are hard
+ /// to model at compile time. For example, if we have multiple LDS
+ /// instructions in flight, one of these may take longer than the other due to
+ /// a bank conflict. Given that these bank conflicts can occur across waves,
+ /// it is hard to accurately model this. This is compounded given the presence
+ /// of the LDS FIFO -- if an earlier LDS instruction has, for example, a bank
+ /// conflict, this will impact the latency of the current LDS instruction.
+ ///
+ /// Given these complexities, we offer different DSLatencyModes for kernels to
+ /// express the average latency for LDS instructions in the kernel. We expect
+ /// that for kernels with many closely packed LDS isntructions, the average
+ /// latency for LDS instructions will be relatively high. Thus we should use
+ /// DSLatencyMode::Loaded or DSLatencyMode::Overloaded.
enum class DSLatencyMode {
- Fast, ///< Use default/pinned latency (no contention)
- Loaded, ///< Use loaded latency (moderate contention, 3x latency)
- Overloaded ///< Use overloaded latency (high contention, 5x latency)
+ Fast, // Use default/pinged latency (no contention)
+ Loaded, // Use loaded latency (moderate contention, 3x latency)
+ Overloaded // Use overloaded latency (high contention, 5x latency)
};
/// \p returns the DS instruction latency multiplier based on the selected
/// DSLatencyMode. \p returns 1 if the default
- /// scheduling model latency should be used (fast mode).
+ /// scheduling model latency should be used (pinged mode).
/// Checks the function attribute first, then if using coexec scheduler
/// defaults to "loaded", then falls back to the global command line option.
static unsigned getDSLatencyMultiplier(const MachineFunction &MF);
@@ -1824,6 +1836,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
unsigned *PredCost = nullptr) const override;
unsigned getBlockingCycles(const MachineInstr &MI) const;
+ /// \returns the latency of a given instruction \p MI. This implements custom
+ /// overrides for certain cases (e.g. when using dfifferent DSLatencyModes to
+ /// express increased LDS resource contention).
unsigned getInstrLatency(const MachineInstr &MI) const;
/// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
>From 3a34205552531f4e347ab8468e89e1ef259051ed Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 15 Sep 2026 20:31:23 -0700
Subject: [PATCH 3/5] Fix lit
Change-Id: Id8c3f63acdce212f1030e5dfc7d9273c56945958
---
.../CodeGen/AMDGPU/coexec-sched-agpr-excess.mir | 4 ++--
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 4 ----
llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir | 6 +++---
llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir | 14 +++++++++++---
4 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
index 7e65714c45d18e..a5d41ebbb8ff72 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
@@ -25,8 +25,8 @@ body: |
; LIMIT-NEXT: %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3)
; LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
; LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
- ; LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
; LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
; LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3)
; LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3)
; LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
@@ -38,9 +38,9 @@ body: |
; NO_LIMIT: %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3)
- ; NO_LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; NO_LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3)
; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec
; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index c6efd5b0ca0bed..88cb777095229a 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -321,10 +321,6 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
-; COEXEC-NEXT: s_wait_dscnt 0x1c
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; COEXEC-NEXT: s_wait_dscnt 0x18
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x14
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x10
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
index 59bc0c0554c9c9..daa95278e30d16 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
@@ -3,13 +3,13 @@
# CHECK: Region: test_ds_latency_fast
-# CHECK: DS: 80 cycles, 4 instrs
+# CHECK: DS: 5 cycles, 4 instrs
# CHECK: Region: test_ds_latency_loaded
-# CHECK: DS: 240 cycles, 4 instrs
+# CHECK: DS: 15 cycles, 4 instrs
# CHECK: Region: test_ds_latency_overloaded
-# CHECK: DS: 400 cycles, 4 instrs
+# CHECK: DS: 25 cycles, 4 instrs
--- |
define void @test_ds_latency_fast() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="fast" { ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
index 527390abcc8f58..50c304a13d9c3f 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
@@ -5,14 +5,22 @@
# Test that the -amdgpu-ds-latency-mode command line flag affects DS cycle
# accounting in the coexec scheduler.
+
# DS instructions have base latency of 20 cycles each (4 instructions = 80 cycles).
# - fast: 4 * 20 * 1 = 80 cycles
# - loaded: 4 * 20 * 3 = 240 cycles
# - overloaded: 4 * 20 * 5 = 400 cycles
-# FAST: DS: 80 cycles, 4 instrs
-# LOADED: DS: 240 cycles, 4 instrs
-# OVERLOADED: DS: 400 cycles, 4 instrs
+# However, the total latency is normalized against the FIFO size (by simple division). For gfx1250, this FIFO size is 16
+
+# - fast: 80 / 16 = 5 cycles
+# - loaded: 240 / 16 = 15 cycles
+# - overloaded: 400 / 16 = 25 cycles
+
+
+# FAST: DS: 5 cycles, 4 instrs
+# LOADED: DS: 15 cycles, 4 instrs
+# OVERLOADED: DS: 25 cycles, 4 instrs
---
name: test_ds_latency_flag
>From 8da7e066d39175ab75586740b3bfd071c6db99d3 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 29 Sep 2026 14:43:39 -0700
Subject: [PATCH 4/5] Rebase
Change-Id: I2629315a64ca3b49d07c5c38444a95027f875282
---
llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir | 2 +-
llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir | 6 +++---
.../CodeGen/AMDGPU/coexec-sched-flavor-classification.mir | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
index d5c03805f294cc..19f4464c4a0cd3 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
@@ -2,7 +2,7 @@
# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler %s -filetype=null 2>&1 | FileCheck --check-prefix=DEBUG %s
# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck --check-prefix=ORDER %s
-# DEBUG: Effective stalls: try=4 (ready=0, struct=0, lat=0, carried=0, buffer=4, fence=0) cand=0 (ready=0, struct=0, lat=0, carried=0, buffer=0, fence=0)
+# DEBUG: Effective stalls: try=44 (ready=0, struct=0, lat=0, carried=0, buffer=44, fence=0) cand=0 (ready=0, struct=0, lat=0, carried=0, buffer=0, fence=0)
# ORDER-LABEL: name: ds_fifo_stall
# ORDER: body: |
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
index 6ce1f7a4c508c7..64453c6ced3122 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
@@ -196,8 +196,8 @@ body: |
; COEXEC-NEXT: dead [[DEF4:%[0-9]+]]:sreg_32_xm0_xexec = S_ADD_I32 [[DEF4]], [[DEF3]].sub1, implicit-def dead $scc
; COEXEC-NEXT: dead early-clobber %18:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; COEXEC-NEXT: dead early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
- ; COEXEC-NEXT: ATOMIC_FENCE 4, 2
; COEXEC-NEXT: dead early-clobber %20:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; COEXEC-NEXT: ATOMIC_FENCE 4, 2
; COEXEC-NEXT: S_ENDPGM 0, amdgpu_allvgprs
%0:vgpr_32 = IMPLICIT_DEF
%1:vreg_256_align2 = IMPLICIT_DEF
@@ -303,8 +303,8 @@ body: |
; COEXEC-NEXT: dead [[DEF4:%[0-9]+]]:sreg_32_xm0_xexec = S_ADD_I32 [[DEF4]], [[DEF3]].sub1, implicit-def dead $scc
; COEXEC-NEXT: dead early-clobber %18:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; COEXEC-NEXT: dead early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
- ; COEXEC-NEXT: S_WAIT_TENSORCNT 5, implicit-def dead $tensorcnt, implicit $tensorcnt
; COEXEC-NEXT: dead early-clobber %20:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; COEXEC-NEXT: S_WAIT_TENSORCNT 5, implicit-def dead $tensorcnt, implicit $tensorcnt
; COEXEC-NEXT: S_ENDPGM 0, amdgpu_allvgprs
%0:vgpr_32 = IMPLICIT_DEF
%1:vreg_256_align2 = IMPLICIT_DEF
@@ -409,8 +409,8 @@ body: |
; COEXEC-NEXT: dead [[DEF4:%[0-9]+]]:sreg_32_xm0_xexec = S_ADD_I32 [[DEF4]], [[DEF3]].sub1, implicit-def dead $scc
; COEXEC-NEXT: dead early-clobber %18:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; COEXEC-NEXT: dead early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
- ; COEXEC-NEXT: S_WAIT_DSCNT 5, implicit-def dead $tensorcnt, implicit $tensorcnt
; COEXEC-NEXT: dead early-clobber %20:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 8, 0, [[DEF7]].sub0, [[DEF8]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; COEXEC-NEXT: S_WAIT_DSCNT 5, implicit-def dead $tensorcnt, implicit $tensorcnt
; COEXEC-NEXT: S_ENDPGM 0, amdgpu_allvgprs
%0:vgpr_32 = IMPLICIT_DEF
%1:vreg_256_align2 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
index 7fcfa684a514e5..79295114b0c91a 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
@@ -14,7 +14,7 @@
# GFX1251-DAG: VALU(1c): 4 cycles, 4 instrs
# GFX1251-DAG: TRANS: 2 cycles, 2 instrs
# GFX1251-DAG: VMEM: 3200 cycles, 10 instrs
-# GFX1251-DAG: DS: 5 cycles, 4 instrs
+# GFX1251-DAG: DS: 15 cycles, 4 instrs
# GFX1251-DAG: DMA: 640 cycles, 2 instrs
--- |
>From 6a7262108d0a9dfb9e172e048ea3c4f0d336acfb Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 29 Sep 2026 16:27:13 -0700
Subject: [PATCH 5/5] Change handling and control
Change-Id: I7032fc47390396257ccc399c30c2cb52aa68f50d
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 2 +-
.../AMDGPU/AMDGPUMachineFunctionInfo.cpp | 3 +
.../Target/AMDGPU/AMDGPUMachineFunctionInfo.h | 5 +
.../Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp | 93 ++++++++++-
.../Target/AMDGPU/AMDGPUPerfHintAnalysis.h | 12 +-
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 145 +++++-------------
llvm/lib/Target/AMDGPU/GCNSubtarget.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 75 ++++-----
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 42 ++---
.../Target/AMDGPU/SIMachineFunctionInfo.cpp | 2 +
.../lib/Target/AMDGPU/SIMachineFunctionInfo.h | 2 +
.../AMDGPU/coexec-sched-agpr-excess.mir | 1 +
.../CodeGen/AMDGPU/coexec-sched-ds-fifo.mir | 3 +
.../AMDGPU/coexec-sched-effective-stall.mir | 18 +++
.../coexec-sched-flavor-classification.mir | 3 +
.../CodeGen/AMDGPU/ds-latency-mode-attr.mir | 25 +--
.../AMDGPU/ds-latency-mode-branch-cost.mir | 7 +-
.../ds-latency-mode-default-scheduler.mir | 7 +-
.../CodeGen/AMDGPU/ds-latency-mode-flag.mir | 9 +-
.../lds-contention-multiplier-invalid.ll | 12 ++
.../CodeGen/AMDGPU/perfhint-lds-contention.ll | 116 ++++++++++++++
.../AMDGPU/perfhint-matrix-intrinsics.ll | 110 +++++++++++++
.../AMDGPU/long-branch-reg-all-sgpr-used.ll | 2 +
.../AMDGPU/machine-function-info-after-pei.ll | 1 +
...ine-function-info-long-branch-reg-debug.ll | 1 +
.../machine-function-info-long-branch-reg.ll | 1 +
.../AMDGPU/machine-function-info-no-ir.mir | 4 +
.../MIR/AMDGPU/machine-function-info.ll | 4 +
28 files changed, 500 insertions(+), 207 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-contention-multiplier-invalid.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/perfhint-lds-contention.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/perfhint-matrix-intrinsics.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index dce9f4ccbed9b4..2e9883ed5e1333 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -585,7 +585,7 @@ unsigned CandidateHeuristics::getMaxBlockingCycles(const MCSchedClassDesc *SC,
unsigned CandidateHeuristics::getHWUICyclesForMI(MachineInstr *MI) {
assert(SchedModel && SchedModel->hasInstrSchedModel());
if (MI->mayLoadOrStore())
- return SchedModel->computeInstrLatency(MI, false);
+ return SII->getInstrLatency(*MI);
return getMaxBlockingCycles(SchedModel->resolveSchedClass(MI), MI);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.cpp
index ea8265a2aae124..73c3d3c47ca93f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.cpp
@@ -55,6 +55,9 @@ AMDGPUMachineFunctionInfo::AMDGPUMachineFunctionInfo(const Function &F,
Attribute WaveLimitAttr = F.getFnAttribute("amdgpu-wave-limiter");
WaveLimiter = WaveLimitAttr.getValueAsBool();
+ Attribute LDSContentionAttr = F.getFnAttribute("amdgpu-lds-contention");
+ LDSContention = LDSContentionAttr.getValueAsBool();
+
// FIXME: How is this attribute supposed to interact with statically known
// global sizes?
StringRef S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.h
index c65592bd965ba8..ac86c5d4de7c44 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunctionInfo.h
@@ -67,6 +67,9 @@ class AMDGPUMachineFunctionInfo : public MachineFunctionInfo {
// Kernel may need limited waves per EU for better performance.
bool WaveLimiter = false;
+ // Function has LDS contention (high LDS load to matrix op ratio in loops).
+ bool LDSContention = false;
+
bool HasInitWholeWave = false;
public:
@@ -99,6 +102,8 @@ class AMDGPUMachineFunctionInfo : public MachineFunctionInfo {
bool needsWaveLimiter() const { return WaveLimiter; }
+ bool hasLDSContention() const { return LDSContention; }
+
bool hasInitWholeWave() const { return HasInitWholeWave; }
void setInitWholeWave() { HasInitWholeWave = true; }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
index f8772da6139fe9..ef33623dbd5b6e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
@@ -24,6 +24,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/CycleInfo.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/Support/CommandLine.h"
@@ -53,6 +54,11 @@ static cl::opt<unsigned>
LargeStrideThresh("amdgpu-large-stride-threshold", cl::init(64), cl::Hidden,
cl::desc("Large stride memory access threshold"));
+static cl::opt<unsigned>
+ LDSContentionThreshold("amdgpu-lds-contention-threshold", cl::init(50),
+ cl::Hidden,
+ cl::desc("Function mem bound threshold in %"));
+
STATISTIC(NumMemBound, "Number of functions marked as memory bound");
STATISTIC(NumLimitWave, "Number of functions marked as needing limit wave");
@@ -95,9 +101,12 @@ struct AMDGPUPerfHint {
const SITargetLowering *TLI;
+ CycleInfo CI;
+
AMDGPUPerfHintAnalysis::FuncInfo *visit(const Function &F);
static bool isMemBound(const AMDGPUPerfHintAnalysis::FuncInfo &F);
static bool needLimitWave(const AMDGPUPerfHintAnalysis::FuncInfo &F);
+ static bool hasLDSCont(const AMDGPUPerfHintAnalysis::FuncInfo &F);
bool isIndirectAccess(const Instruction *Inst) const;
@@ -113,6 +122,8 @@ struct AMDGPUPerfHint {
bool isGlobalAddr(const Value *V) const;
bool isLocalAddr(const Value *V) const;
bool isGlobalLoadUsedInBB(const Instruction &) const;
+ bool isLDSLoad(const Instruction *I) const;
+ static bool isMatrixIntrinsic(const Instruction *I);
};
static std::pair<const Value *, const Type *> getMemoryInstrPtrAndType(
@@ -216,10 +227,30 @@ AMDGPUPerfHintAnalysis::FuncInfo *AMDGPUPerfHint::visit(const Function &F) {
LLVM_DEBUG(dbgs() << "[AMDGPUPerfHint] process " << F.getName() << '\n');
- for (auto &B : F) {
+ auto IsInnermostCycle = [this](const BasicBlock &B) {
+ CycleRef C = CI.getCycle(&B);
+ if (!C.isValid())
+ return false;
+
+ return llvm::none_of(CI.getBlocks(C), [&](const BasicBlock *CB) {
+ CycleRef InnerC = CI.getCycle(CB);
+ return InnerC != C && CI.contains(C, InnerC);
+ });
+ };
+
+ for (const BasicBlock &B : F) {
LastAccess = MemAccessInfo();
unsigned UsedGlobalLoadsInBB = 0;
- for (auto &I : B) {
+
+ bool InInnermostCycle = IsInnermostCycle(B);
+
+ for (const Instruction &I : B) {
+ if (InInnermostCycle) {
+ if (isMatrixIntrinsic(&I))
+ ++FI.MatrixInstLoopCost;
+ else if (isLDSLoad(&I))
+ ++FI.LDSInstLoopCost;
+ }
if (const Type *Ty = getMemoryInstrPtrAndType(&I).second) {
unsigned Size = divideCeil(Ty->getPrimitiveSizeInBits(), 32);
// TODO: Check if the global load and its user are close to each other
@@ -277,6 +308,9 @@ AMDGPUPerfHintAnalysis::FuncInfo *AMDGPUPerfHint::visit(const Function &F) {
}
}
+ // Check for LDS contention after visiting all basic blocks
+ FI.HasLDSContention = hasLDSCont(FI);
+
return &FI;
}
@@ -285,9 +319,12 @@ bool AMDGPUPerfHint::runOnFunction(Function &F) {
DL = &M.getDataLayout();
if (F.hasFnAttribute("amdgpu-wave-limiter") &&
- F.hasFnAttribute("amdgpu-memory-bound"))
+ F.hasFnAttribute("amdgpu-memory-bound") &&
+ F.hasFnAttribute("amdgpu-lds-contention"))
return false;
+ CI.compute(F);
+
const AMDGPUPerfHintAnalysis::FuncInfo *Info = visit(F);
LLVM_DEBUG(dbgs() << F.getName() << " MemInst cost: " << Info->MemInstCost
@@ -312,6 +349,11 @@ bool AMDGPUPerfHint::runOnFunction(Function &F) {
Changed = true;
}
+ if (!F.hasFnAttribute("amdgpu-lds-contention") && Info->HasLDSContention) {
+ F.addFnAttr("amdgpu-lds-contention", "true");
+ Changed = true;
+ }
+
return Changed;
}
@@ -324,6 +366,21 @@ bool AMDGPUPerfHint::isMemBound(const AMDGPUPerfHintAnalysis::FuncInfo &FI) {
return FI.MemInstCost * 100 / FI.InstCost > MemBoundThresh;
}
+bool AMDGPUPerfHint::hasLDSCont(const AMDGPUPerfHintAnalysis::FuncInfo &FI) {
+ if (FI.MatrixInstLoopCost > 0 &&
+ FI.LDSInstLoopCost * 100 / FI.MatrixInstLoopCost >=
+ LDSContentionThreshold) {
+ LLVM_DEBUG(
+ dbgs() << "Found cycle with LDS contention: " << FI.LDSInstLoopCost
+ << " LDS loads, " << FI.MatrixInstLoopCost << " matrix ops\n"
+ << "Threshold for LDS/MatrixOp ratio: " << LDSContentionThreshold
+ << "\n");
+ return true;
+ }
+
+ return false;
+}
+
bool AMDGPUPerfHint::needLimitWave(const AMDGPUPerfHintAnalysis::FuncInfo &FI) {
return ((FI.MemInstCost + FI.IAMInstCost * IAWeight +
FI.LSMInstCost * LSWeight) * 100 / FI.InstCost) > LimitWaveThresh;
@@ -344,6 +401,28 @@ bool AMDGPUPerfHint::isLocalAddr(const Value *V) const {
return false;
}
+bool AMDGPUPerfHint::isLDSLoad(const Instruction *I) const {
+ if (const auto *LdI = dyn_cast<LoadInst>(I))
+ return isLocalAddr(LdI->getPointerOperand());
+ if (const auto *II = dyn_cast<IntrinsicInst>(I)) {
+ if (!II->mayReadFromMemory() || II->arg_empty())
+ return false;
+ return isLocalAddr(II->getArgOperand(0));
+ }
+ return false;
+}
+
+bool AMDGPUPerfHint::isMatrixIntrinsic(const Instruction *I) {
+ if (const auto *II = dyn_cast<IntrinsicInst>(I)) {
+ StringRef Name = Intrinsic::getBaseName(II->getIntrinsicID());
+ return Name.starts_with("llvm.amdgcn.wmma.") ||
+ Name.starts_with("llvm.amdgcn.mfma.") ||
+ Name.starts_with("llvm.amdgcn.swmmac.") ||
+ Name.starts_with("llvm.amdgcn.smfmac.");
+ }
+ return false;
+}
+
bool AMDGPUPerfHint::isLargeStride(const Instruction *Inst) {
LLVM_DEBUG(dbgs() << "[isLargeStride] " << *Inst << '\n');
@@ -420,6 +499,14 @@ bool AMDGPUPerfHintAnalysis::needsWaveLimiter(const Function *F) const {
return AMDGPUPerfHint::needLimitWave(FI->second);
}
+bool AMDGPUPerfHintAnalysis::hasLDSContention(const Function *F) const {
+ auto FI = FIM.find(F);
+ if (FI == FIM.end())
+ return false;
+
+ return FI->second.HasLDSContention;
+}
+
bool AMDGPUPerfHintAnalysis::runOnSCC(const GCNTargetMachine &TM,
CallGraphSCC &SCC) {
bool Changed = false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h
index b7880c4639a977..533933696084fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h
@@ -33,13 +33,17 @@ class AMDGPUPerfHintAnalysis {
struct FuncInfo {
unsigned MemInstCost;
unsigned InstCost;
- unsigned IAMInstCost; // Indirect access memory instruction count
- unsigned LSMInstCost; // Large stride memory instruction count
+ unsigned IAMInstCost; // Indirect access memory instruction count
+ unsigned LSMInstCost; // Large stride memory instruction count
+ unsigned LDSInstLoopCost; // LDS load in-loop instruction count
+ unsigned MatrixInstLoopCost; // matmul intrinsic in-loop instruction count
bool HasDenseGlobalMemAcc; // Set if at least 1 basic block has relatively
// high global memory access
+ bool HasLDSContention; // Set if any loop has LDS loads >= 0.5 * WMMA/MFMA
FuncInfo()
: MemInstCost(0), InstCost(0), IAMInstCost(0), LSMInstCost(0),
- HasDenseGlobalMemAcc(false) {}
+ LDSInstLoopCost(0), MatrixInstLoopCost(0),
+ HasDenseGlobalMemAcc(false), HasLDSContention(false) {}
};
typedef ValueMap<const Function *, FuncInfo> FuncInfoMap;
@@ -59,6 +63,8 @@ class AMDGPUPerfHintAnalysis {
bool isMemoryBound(const Function *F) const;
bool needsWaveLimiter(const Function *F) const;
+
+ bool hasLDSContention(const Function *F) const;
};
struct AMDGPUPerfHintAnalysisPass
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 894e632f129128..3f0c79aac945fb 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -3427,7 +3427,7 @@ int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
break;
}
- int NumPasses = TSchedModel.computeInstrLatency(Writer);
+ int NumPasses = TII.getInstrLatency(*Writer);
if (ST.hasGFX940Insts()) {
if (!TII.isXDL(*Writer))
return GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
@@ -3482,8 +3482,7 @@ int GCNHazardRecognizer::getMFMAReadWaitStates(const MachineInstr &Consumer,
(Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
- else if (ST.hasGFX940Insts() &&
- TSchedModel.computeInstrLatency(&Producer) == 2)
+ else if (ST.hasGFX940Insts() && TII.getInstrLatency(Producer) == 2)
NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
// The accumulator forwarding path that allows zero wait states is only
@@ -3513,7 +3512,7 @@ int GCNHazardRecognizer::getMFMAReadWaitStates(const MachineInstr &Consumer,
NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
break;
default:
- int NumPasses = TSchedModel.computeInstrLatency(&Producer);
+ int NumPasses = TII.getInstrLatency(Producer);
if (ST.hasGFX940Insts()) {
NeedWaitStates =
@@ -3584,81 +3583,15 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
int OpNo = Use.getOperandNo();
- unsigned Opc1 = MI1->getOpcode();
- int NeedWaitStates = 0;
- if (OpNo == SrcCIdx) {
- if (!SIInstrInfo::isDGEMM(Opc) &&
- (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
- NeedWaitStates = 0;
- } else if (FullReg) {
- if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
- Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
- (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
- Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
- NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
- else if (ST.hasGFX940Insts() && TII.getInstrLatency(*MI1) == 2)
- NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
-
- // The accumulator forwarding path that allows zero wait states is only
- // available while the chain stays on a single MFMA. Two different MFMAs
- // sharing an accumulator need the wait states of a partial overlap.
- if (ST.hasGFX940Insts() && !isSameMFMA(Opc, Opc1)) {
- NeedWaitStates = std::max(NeedWaitStates,
- getMFMAOverlappedSrcCWaitStates(MI, MI1));
- }
- } else {
- NeedWaitStates = getMFMAOverlappedSrcCWaitStates(MI, MI1);
- }
- } else {
- switch (Opc1) {
- case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
- case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
- case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
- case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
- NeedWaitStates =
- ST.hasGFX950Insts()
- ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
- : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
- break;
- case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
- case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
- NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
- break;
- default:
- int NumPasses = TII.getInstrLatency(*MI1);
-
- if (ST.hasGFX940Insts()) {
- NeedWaitStates =
- TII.isXDL(*MI1)
- ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
- NumPasses, ST.hasGFX950Insts())
- : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
- NumPasses);
- break;
- }
-
- switch (NumPasses) {
- case 2:
- NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
- break;
- case 4:
- llvm_unreachable("unexpected number of passes for mfma");
- case 8:
- NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
- break;
- case 16:
- default:
- NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
- }
- }
- }
- assert(NeedWaitStates <= MaxWaitStates &&
- "hazard requirement exceeds the scan window");
- if (WaitStatesNeeded >= NeedWaitStates)
- continue;
-
- WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
- WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+ WaitStatesNeeded = std::max(
+ WaitStatesNeeded,
+ getMaxWindowDeficit(
+ MaxWaitStates, [&](const MachineInstr &P) -> std::optional<int> {
+ if (!SIInstrInfo::isMFMA(P) ||
+ !TRI.regsOverlap(P.getOperand(0).getReg(), Reg))
+ return std::nullopt;
+ return getMFMAReadWaitStates(*MI, P, Reg, OpNo == SrcCIdx);
+ }));
if (WaitStatesNeeded == MaxWaitStates)
break;
@@ -3902,21 +3835,29 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
}
auto MFMAWindow = [&](const MachineInstr &Producer) {
- int NumPasses = TSchedModel.computeInstrLatency(&Producer);
+ int NumPasses = TII.getInstrLatency(Producer);
int NeedWaitStates = MaxWaitStates;
- unsigned HazardDefLatency = TII.getInstrLatency(*MFMA);
- int NumPasses = HazardDefLatency;
- int NeedWaitStates = MaxWaitStates;
-
- if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
- switch (HazardDefLatency) {
- case 4:
- NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
- : DMFMA4x4WriteVgprVALUReadWaitStates;
- break;
- case 8:
- case 16:
+ if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
+ switch (NumPasses) {
+ case 4:
+ NeedWaitStates = IsMemOrExport
+ ? DMFMA4x4WriteVgprMemExpReadWaitStates
+ : DMFMA4x4WriteVgprVALUReadWaitStates;
+ break;
+ case 8:
+ case 16:
+ NeedWaitStates =
+ IsMemOrExport
+ ? DMFMA16x16WriteVgprMemExpReadWaitStates
+ : (ST.hasGFX950Insts()
+ ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
+ : DMFMA16x16WriteVgprVALUReadWaitStates);
+ break;
+ default:
+ llvm_unreachable("unexpected dgemm");
+ }
+ } else if (ST.hasGFX940Insts()) {
NeedWaitStates =
TII.isXDL(Producer)
? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
@@ -3998,7 +3939,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
auto MFMAWawWindow = [&](const MachineInstr &Producer) {
int NeedWaitStates = MaxWaitStates;
- int NumPasses = TII.getInstrLatency(*MFMA);
+ int NumPasses = TII.getInstrLatency(Producer);
if (SIInstrInfo::isDGEMM(Producer.getOpcode())) {
switch (NumPasses) {
@@ -4048,20 +3989,8 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
return MFMAWawWindow(P);
}));
- unsigned HazardDefLatency = TII.getInstrLatency(*MFMA);
- int NeedWaitStates = MaxWaitStates;
- switch (HazardDefLatency) {
- case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
- break;
- case 4: assert(ST.hasGFX940Insts());
- NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
- break;
- case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
- break;
- case 16: [[fallthrough]];
- default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
- break;
- }
+ if (WaitStatesNeeded >= MaxWaitStates)
+ break;
// Wait states this write needs after an MFMA that read Reg as srcC.
auto SMFMAReadAsCWindow =
@@ -4080,7 +4009,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
return std::nullopt;
- switch (TSchedModel.computeInstrLatency(&Reader)) {
+ switch (TII.getInstrLatency(Reader)) {
case 2:
return SMFMA4x4ReadVgprVALUWarWaitStates;
case 4:
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 2ad02f053eb5ae..2fb9d4ace16222 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -854,7 +854,7 @@ void GCNSubtarget::adjustSchedDependency(
// DS load/store latency is variable depending on LDS contention.
if (InstrInfo.isDS(*DefI) &&
- InstrInfo.getDSLatencyMultiplier(*DefI->getMF()) != 1) {
+ InstrInfo.getDSLatencyMultiplier(*DefI->getMF()) != 100) {
// For LDS instructions, we have overrides to change default latencies.
unsigned Latency = InstrInfo.getInstrLatency(*DefI);
Dep.setLatency(Latency);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3da21a26efc980..3245752d7199cb 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -71,16 +71,31 @@ static cl::opt<bool> Fix16BitCopies(
cl::init(true),
cl::ReallyHidden);
-static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency(
- "amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"),
- cl::values(
- clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast",
- "Use default/pinged latency (no contention)"),
- clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded",
- "Use loaded latency (moderate contention, 3x latency)"),
- clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded",
- "Use overloaded latency (high contention, 5x latency)")),
- cl::init(SIInstrInfo::DSLatencyMode::Fast), cl::Hidden);
+namespace {
+
+struct LDSContentionMultiplierParser : public cl::parser<unsigned> {
+ LDSContentionMultiplierParser(cl::Option &O) : cl::parser<unsigned>(O) {}
+
+ bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
+ if (Arg.getAsInteger(0, Value))
+ return O.error("'" + Arg + "' value invalid for uint argument!");
+
+ if (Value == 0)
+ return O.error("'" + Arg + "' value must be greater than 0!");
+
+ return false;
+ }
+};
+
+} // end anonymous namespace
+
+static cl::opt<unsigned, false, LDSContentionMultiplierParser>
+ LDSContentionMultiplier("amdgpu-lds-contention-multiplier",
+ cl::ReallyHidden, cl::init(300),
+ cl::desc("How much to scale the latency of LDS "
+ "instructions in kernels with high LDS "
+ "contention. Specified as a percent "
+ "(e.g. 300 = 3x, 50 = 0.5x)"));
SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
: AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
@@ -11256,7 +11271,9 @@ unsigned SIInstrInfo::getInstrLatency(const MachineInstr &MI) const {
if (SchedModel.hasInstrSchedModel()) {
unsigned Latency = SchedModel.computeInstrLatency(&MI);
if (isDS(MI)) {
- Latency *= getDSLatencyMultiplier(*MI.getMF());
+ unsigned Multiplier = getDSLatencyMultiplier(*MI.getMF());
+ if (Multiplier != 100)
+ Latency = Latency * Multiplier / 100;
}
return Latency;
}
@@ -12010,32 +12027,16 @@ bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
unsigned SIInstrInfo::getDSLatencyMultiplier(const MachineFunction &MF) {
const Function &F = MF.getFunction();
- // Priority selection goes to the attribute
- Attribute A = F.getFnAttribute("amdgpu-ds-latency-mode");
- if (A.isValid()) {
- StringRef Val = A.getValueAsString();
- if (Val == "fast")
- return 1;
- if (Val == "loaded")
- return 3;
- if (Val == "overloaded")
- return 5;
- }
-
- // If using coexec scheduler, default to "loaded" mode unless overridden
- // by the command line option.
- if (DSLatency.getNumOccurrences() == 0 &&
- AMDGPU::getSchedStrategy(F) == "coexec")
- return 3;
+ // Only apply latency multiplier for coexec scheduler
+ if (AMDGPU::getSchedStrategy(F) != "coexec" &&
+ !LDSContentionMultiplier.getNumOccurrences())
+ return 100;
- switch (DSLatency) {
- case DSLatencyMode::Fast:
- return 1; // Use default scheduling model latency
- case DSLatencyMode::Loaded:
- return 3;
- case DSLatencyMode::Overloaded:
- return 5;
- }
+ // Check for LDS contention detected by AMDGPUPerfHintAnalysis
+ const AMDGPUMachineFunctionInfo *MFI =
+ MF.getInfo<AMDGPUMachineFunctionInfo>();
+ if (MFI->hasLDSContention())
+ return LDSContentionMultiplier;
- return 1;
+ return 100;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 5bd5f39981b8f6..170fb79b466419 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1261,31 +1261,21 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
}
- /// DS latency modes. The latency of an individual DS load/store instruction
- /// is variable. Some of the major sources that cause this variation are hard
- /// to model at compile time. For example, if we have multiple LDS
- /// instructions in flight, one of these may take longer than the other due to
- /// a bank conflict. Given that these bank conflicts can occur across waves,
- /// it is hard to accurately model this. This is compounded given the presence
- /// of the LDS FIFO -- if an earlier LDS instruction has, for example, a bank
- /// conflict, this will impact the latency of the current LDS instruction.
+ /// The latency of an individual DS load/store instruction is variable.
+ /// Some of the major sources that cause this variation are hard
+ /// to model at compile time.
///
- /// Given these complexities, we offer different DSLatencyModes for kernels to
- /// express the average latency for LDS instructions in the kernel. We expect
- /// that for kernels with many closely packed LDS isntructions, the average
- /// latency for LDS instructions will be relatively high. Thus we should use
- /// DSLatencyMode::Loaded or DSLatencyMode::Overloaded.
- enum class DSLatencyMode {
- Fast, // Use default/pinged latency (no contention)
- Loaded, // Use loaded latency (moderate contention, 3x latency)
- Overloaded // Use overloaded latency (high contention, 5x latency)
- };
-
- /// \p returns the DS instruction latency multiplier based on the selected
- /// DSLatencyMode. \p returns 1 if the default
- /// scheduling model latency should be used (pinged mode).
- /// Checks the function attribute first, then if using coexec scheduler
- /// defaults to "loaded", then falls back to the global command line option.
+ /// Given these complexities, we do not attempt to model the latency, and
+ /// rather use a reasonable latency multiplier when there is LDS contention in
+ /// the kernel. While there are overrides to this multiplier, it is expected
+ /// that users can design kernels in such a way to acheive this latency on HW.
+ /// The compiler uses a simple heuristic to detect if the kernel has LDS
+ /// contention.
+ ///
+ /// Returns the DS latency multiplier as a percentage. If LDS contention was
+ /// detected by AMDGPUPerfHintAnalysis and using coexec scheduler, returns
+ /// amdgpu-lds-contention-multiplier (default 300 = 3x). Otherwise returns
+ /// 100 (no scaling).
static unsigned getDSLatencyMultiplier(const MachineFunction &MF);
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
@@ -1837,8 +1827,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
unsigned getBlockingCycles(const MachineInstr &MI) const;
/// \returns the latency of a given instruction \p MI. This implements custom
- /// overrides for certain cases (e.g. when using dfifferent DSLatencyModes to
- /// express increased LDS resource contention).
+ /// overrides for certain cases (e.g. increased LDS latency when contention
+ /// is detected).
unsigned getInstrLatency(const MachineInstr &MI) const;
/// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 1cef1223c246cd..5728cf34f4534a 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -730,6 +730,7 @@ yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),
IsEntryFunction(MFI.isEntryFunction()), MemoryBound(MFI.isMemoryBound()),
WaveLimiter(MFI.needsWaveLimiter()),
+ LDSContention(MFI.hasLDSContention()),
HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()),
@@ -790,6 +791,7 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields(
IsEntryFunction = YamlMFI.IsEntryFunction;
MemoryBound = YamlMFI.MemoryBound;
WaveLimiter = YamlMFI.WaveLimiter;
+ LDSContention = YamlMFI.LDSContention;
HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback;
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 359fc36dbcc714..f073b1bd386990 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -270,6 +270,7 @@ struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
bool IsChainFunction = false;
bool MemoryBound = false;
bool WaveLimiter = false;
+ bool LDSContention = false;
bool HasSpilledSGPRs = false;
bool HasSpilledVGPRs = false;
bool HasNoWWMPoolSGPRSpillFallback = false;
@@ -333,6 +334,7 @@ template <> struct MappingTraits<SIMachineFunctionInfo> {
YamlIO.mapOptional("isChainFunction", MFI.IsChainFunction, false);
YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
+ YamlIO.mapOptional("hasLDSContention", MFI.LDSContention, false);
YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
YamlIO.mapOptional("hasNoWWMPoolSGPRSpillFallback",
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
index a5d41ebbb8ff72..666e9e83e5d67b 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir
@@ -17,6 +17,7 @@ name: agpr_excess_tracking
tracksRegLiveness: true
machineFunctionInfo:
isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; LIMIT-LABEL: name: agpr_excess_tracking
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
index 19f4464c4a0cd3..9f2e4bc78e2343 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
@@ -23,6 +23,9 @@
---
name: ds_fifo_stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
%0:vgpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
index 64453c6ced3122..12ba96d0bb40a8 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
@@ -16,6 +16,9 @@
---
name: test-sched-effective-stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; DEFAULT-LABEL: name: test-sched-effective-stall
@@ -74,6 +77,9 @@ body: |
---
name: test-sched-pending-structural-stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; DEFAULT-LABEL: name: test-sched-pending-structural-stall
@@ -130,6 +136,9 @@ body: |
---
name: test-fence-stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; DEFAULT-LABEL: name: test-fence-stall
@@ -237,6 +246,9 @@ body: |
---
name: test-tensorcnt-stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; DEFAULT-LABEL: name: test-tensorcnt-stall
@@ -343,6 +355,9 @@ body: |
---
name: test-dscnt-stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; DEFAULT-LABEL: name: test-dscnt-stall
@@ -449,6 +464,9 @@ body: |
---
name: test-memory-pipeline-effective-stall
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; DEFAULT-LABEL: name: test-memory-pipeline-effective-stall
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
index 79295114b0c91a..6af3eda9a6d0b5 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
@@ -44,6 +44,9 @@
---
name: test_flavor_classification
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
%0:vreg_64_align2 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
index daa95278e30d16..5094f557d87c80 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
@@ -8,14 +8,9 @@
# CHECK: Region: test_ds_latency_loaded
# CHECK: DS: 15 cycles, 4 instrs
-# CHECK: Region: test_ds_latency_overloaded
-# CHECK: DS: 25 cycles, 4 instrs
-
--- |
- define void @test_ds_latency_fast() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="fast" { ret void }
- define void @test_ds_latency_loaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="loaded" { ret void }
- define void @test_ds_latency_overloaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="overloaded" { ret void }
-
+ define void @test_ds_latency_fast() "amdgpu-waves-per-eu"="1,1" { ret void }
+ define void @test_ds_latency_loaded() "amdgpu-waves-per-eu"="1,1" { ret void }
...
---
@@ -35,20 +30,8 @@ body: |
---
name: test_ds_latency_loaded
tracksRegLiveness: true
-body: |
- bb.0:
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vreg_64_align2 = IMPLICIT_DEF
- %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
- %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
- DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
- DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
- S_ENDPGM 0, implicit %2, implicit %3
-...
-
----
-name: test_ds_latency_overloaded
-tracksRegLiveness: true
+machineFunctionInfo:
+ hasLDSContention: true
body: |
bb.0:
%0:vgpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
index 95bf9df77965a8..2c32a32ea20a1a 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-lds-contention-multiplier=100 %s -o - | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-lds-contention-multiplier=500 %s -o - | FileCheck -check-prefix=OVERLOADED %s
# Test that ds-latency-mode affects the branch-over-removal cost model in
# SIPreEmitPeephole. With higher DS latency, the cost of the then-block
@@ -8,6 +8,9 @@
---
name: skip_execz_ds_multi
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
; FAST-LABEL: name: skip_execz_ds_multi
; FAST: bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
index 41695b863b00a5..e94d9e84ae8372 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
@@ -1,11 +1,14 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-lds-contention-multiplier=100 %s -o - | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-lds-contention-multiplier=500 %s -o - | FileCheck -check-prefix=OVERLOADED %s
---
name: test_ds_latency_default_sched
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
; FAST-LABEL: name: test_ds_latency_default_sched
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
index 50c304a13d9c3f..46d38220cb2d22 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
@@ -1,7 +1,7 @@
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=fast %s -filetype=null 2>&1 | FileCheck -check-prefix=FAST %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=loaded %s -filetype=null 2>&1 | FileCheck -check-prefix=LOADED %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -filetype=null 2>&1 | FileCheck -check-prefix=OVERLOADED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-lds-contention-multiplier=100 %s -filetype=null 2>&1 | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-lds-contention-multiplier=300 %s -filetype=null 2>&1 | FileCheck -check-prefix=LOADED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-lds-contention-multiplier=500 %s -filetype=null 2>&1 | FileCheck -check-prefix=OVERLOADED %s
# Test that the -amdgpu-ds-latency-mode command line flag affects DS cycle
# accounting in the coexec scheduler.
@@ -25,6 +25,9 @@
---
name: test_ds_latency_flag
tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ hasLDSContention: true
body: |
bb.0:
%0:vgpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/lds-contention-multiplier-invalid.ll b/llvm/test/CodeGen/AMDGPU/lds-contention-multiplier-invalid.ll
new file mode 100644
index 00000000000000..4ae6cd24c060e9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-contention-multiplier-invalid.ll
@@ -0,0 +1,12 @@
+; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-lds-contention-multiplier=0 -filetype=null %s 2>&1 | FileCheck --check-prefix=CHECK-ZERO %s
+; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-lds-contention-multiplier=-1 -filetype=null %s 2>&1 | FileCheck --check-prefix=CHECK-NEG %s
+
+; Verify that zero value is rejected by the parser
+; CHECK-ZERO: '0' value must be greater than 0!
+
+; Verify that negative values are rejected by the command line parser
+; CHECK-NEG: '-1' value invalid for uint argument!
+
+define void @test() {
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/perfhint-lds-contention.ll b/llvm/test/CodeGen/AMDGPU/perfhint-lds-contention.ll
new file mode 100644
index 00000000000000..574c288945d1d0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/perfhint-lds-contention.ll
@@ -0,0 +1,116 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-perf-hint < %s | FileCheck %s
+
+; Test that AMDGPUPerfHintsAnalysis sets the amdgpu-lds-contention attribute
+; based on the ratio of LDS loads to matrix operations in innermost loops.
+; Contention is flagged when: LDSLoadCount * 2 >= MatrixInstCount
+
+; High LDS ratio: 2 LDS loads, 1 matrix op -> 2*2=4 >= 1, contention=true
+; CHECK-LABEL: define void @high_lds_ratio(
+; CHECK-SAME: #[[HIGH_ATTR:[0-9]+]]
+define void @high_lds_ratio(ptr addrspace(3) %lds, float %a, float %b, <32 x float> %c, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <32 x float> [%c, %entry], [%result, %loop]
+ %ptr1 = getelementptr float, ptr addrspace(3) %lds, i32 %i
+ %lds.val1 = load float, ptr addrspace(3) %ptr1
+ %i2 = add i32 %i, 1
+ %ptr2 = getelementptr float, ptr addrspace(3) %lds, i32 %i2
+ %lds.val2 = load float, ptr addrspace(3) %ptr2
+ %sum = fadd float %lds.val1, %lds.val2
+ %result = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %sum, float %b, <32 x float> %acc, i32 0, i32 0, i32 0)
+ %i.next = add i32 %i, 2
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <32 x float> %result, ptr addrspace(1) %out
+ ret void
+}
+
+; Low LDS ratio: 1 LDS load, 4 matrix ops -> 1*2=2 < 4, contention=false
+; CHECK-LABEL: define void @low_lds_ratio(
+; CHECK-SAME: #[[LOW_ATTR:[0-9]+]]
+define void @low_lds_ratio(ptr addrspace(3) %lds, float %a, float %b, <32 x float> %c, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <32 x float> [%c, %entry], [%result4, %loop]
+ %ptr1 = getelementptr float, ptr addrspace(3) %lds, i32 %i
+ %lds.val = load float, ptr addrspace(3) %ptr1
+ %result1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %lds.val, float %b, <32 x float> %acc, i32 0, i32 0, i32 0)
+ %result2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %lds.val, float %b, <32 x float> %result1, i32 0, i32 0, i32 0)
+ %result3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %lds.val, float %b, <32 x float> %result2, i32 0, i32 0, i32 0)
+ %result4 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %lds.val, float %b, <32 x float> %result3, i32 0, i32 0, i32 0)
+ %i.next = add i32 %i, 1
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <32 x float> %result4, ptr addrspace(1) %out
+ ret void
+}
+
+; Boundary case: 1 LDS load, 2 matrix ops -> 1*2=2 >= 2, contention=true
+; CHECK-LABEL: define void @boundary_lds_ratio(
+; CHECK-SAME: #[[BOUNDARY_ATTR:[0-9]+]]
+define void @boundary_lds_ratio(ptr addrspace(3) %lds, float %a, float %b, <32 x float> %c, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <32 x float> [%c, %entry], [%result2, %loop]
+ %ptr1 = getelementptr float, ptr addrspace(3) %lds, i32 %i
+ %lds.val = load float, ptr addrspace(3) %ptr1
+ %result1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %lds.val, float %b, <32 x float> %acc, i32 0, i32 0, i32 0)
+ %result2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %lds.val, float %b, <32 x float> %result1, i32 0, i32 0, i32 0)
+ %i.next = add i32 %i, 1
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <32 x float> %result2, ptr addrspace(1) %out
+ ret void
+}
+
+; User-specified amdgpu-lds-contention="false" should not be overridden by heuristic
+; CHECK-LABEL: define void @user_override_false(
+; CHECK-SAME: #[[USER_FALSE_ATTR:[0-9]+]]
+define void @user_override_false(ptr addrspace(3) %lds, float %a, float %b, <32 x float> %c, ptr addrspace(1) %out) #0 {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <32 x float> [%c, %entry], [%result, %loop]
+ %ptr1 = getelementptr float, ptr addrspace(3) %lds, i32 %i
+ %lds.val1 = load float, ptr addrspace(3) %ptr1
+ %i2 = add i32 %i, 1
+ %ptr2 = getelementptr float, ptr addrspace(3) %lds, i32 %i2
+ %lds.val2 = load float, ptr addrspace(3) %ptr2
+ %sum = fadd float %lds.val1, %lds.val2
+ %result = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %sum, float %b, <32 x float> %acc, i32 0, i32 0, i32 0)
+ %i.next = add i32 %i, 2
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <32 x float> %result, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = { "amdgpu-lds-contention"="false" }
+
+; High ratio and boundary case should have lds-contention=true (same attribute group)
+; CHECK-DAG: attributes #[[HIGH_ATTR]]{{.*}}"amdgpu-lds-contention"="true"
+; Low ratio should have memory-bound but NOT lds-contention
+; CHECK-DAG: attributes #[[LOW_ATTR]] = { "amdgpu-memory-bound"="true" }
+; User-specified false should be preserved (not overridden to true)
+; CHECK-DAG: attributes #[[USER_FALSE_ATTR]] = { "amdgpu-lds-contention"="false"
+
+declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/perfhint-matrix-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/perfhint-matrix-intrinsics.ll
new file mode 100644
index 00000000000000..d2fb7a9db6740b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/perfhint-matrix-intrinsics.ll
@@ -0,0 +1,110 @@
+; REQUIRES: asserts
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-perf-hint --debug-only=amdgpu-perf-hint < %s 2>&1 | FileCheck %s
+
+; Test that all matrix intrinsic families (wmma, mfma, swmmac, smfmac) are
+; recognized by the perf hint analysis for LDS contention detection.
+
+; CHECK: [AMDGPUPerfHint] process test_wmma_contention
+; CHECK: Found cycle with LDS contention: 2 LDS loads, 1 matrix ops
+define void @test_wmma_contention(ptr addrspace(3) %lds, <16 x half> %a, <16 x half> %b, <16 x half> %c, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <16 x half> [%c, %entry], [%result, %loop]
+ %ptr1 = getelementptr <16 x half>, ptr addrspace(3) %lds, i32 %i
+ %lds.val1 = load <16 x half>, ptr addrspace(3) %ptr1
+ %i2 = add i32 %i, 1
+ %ptr2 = getelementptr <16 x half>, ptr addrspace(3) %lds, i32 %i2
+ %lds.val2 = load <16 x half>, ptr addrspace(3) %ptr2
+ %sum = fadd <16 x half> %lds.val1, %lds.val2
+ %result = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %sum, <16 x half> %b, <16 x half> %acc, i1 false)
+ %i.next = add i32 %i, 2
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <16 x half> %result, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK: [AMDGPUPerfHint] process test_mfma_contention
+; CHECK: Found cycle with LDS contention: 2 LDS loads, 1 matrix ops
+define void @test_mfma_contention(ptr addrspace(3) %lds, float %a, float %b, <32 x float> %c, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <32 x float> [%c, %entry], [%result, %loop]
+ %ptr1 = getelementptr float, ptr addrspace(3) %lds, i32 %i
+ %lds.val1 = load float, ptr addrspace(3) %ptr1
+ %i2 = add i32 %i, 1
+ %ptr2 = getelementptr float, ptr addrspace(3) %lds, i32 %i2
+ %lds.val2 = load float, ptr addrspace(3) %ptr2
+ %sum = fadd float %lds.val1, %lds.val2
+ %result = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %sum, float %b, <32 x float> %acc, i32 0, i32 0, i32 0)
+ %i.next = add i32 %i, 2
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <32 x float> %result, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK: [AMDGPUPerfHint] process test_swmmac_contention
+; CHECK: Found cycle with LDS contention: 2 LDS loads, 1 matrix ops
+define void @test_swmmac_contention(ptr addrspace(3) %lds, <8 x half> %a, <16 x half> %b, <8 x float> %c, i16 %idx, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <8 x float> [%c, %entry], [%result, %loop]
+ %ptr1 = getelementptr <8 x half>, ptr addrspace(3) %lds, i32 %i
+ %lds.val1 = load <8 x half>, ptr addrspace(3) %ptr1
+ %i2 = add i32 %i, 1
+ %ptr2 = getelementptr <8 x half>, ptr addrspace(3) %lds, i32 %i2
+ %lds.val2 = load <8 x half>, ptr addrspace(3) %ptr2
+ %sum = fadd <8 x half> %lds.val1, %lds.val2
+ %result = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16(<8 x half> %sum, <16 x half> %b, <8 x float> %acc, i16 %idx)
+ %i.next = add i32 %i, 2
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <8 x float> %result, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK: [AMDGPUPerfHint] process test_smfmac_contention
+; CHECK: Found cycle with LDS contention: 2 LDS loads, 1 matrix ops
+define void @test_smfmac_contention(ptr addrspace(3) %lds, <4 x half> %a, <8 x half> %b, <4 x float> %c, ptr addrspace(1) %out) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [0, %entry], [%i.next, %loop]
+ %acc = phi <4 x float> [%c, %entry], [%result, %loop]
+ %ptr1 = getelementptr <4 x half>, ptr addrspace(3) %lds, i32 %i
+ %lds.val1 = load <4 x half>, ptr addrspace(3) %ptr1
+ %i2 = add i32 %i, 1
+ %ptr2 = getelementptr <4 x half>, ptr addrspace(3) %lds, i32 %i2
+ %lds.val2 = load <4 x half>, ptr addrspace(3) %ptr2
+ %sum = fadd <4 x half> %lds.val1, %lds.val2
+ %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half> %sum, <8 x half> %b, <4 x float> %acc, i32 0, i32 0, i32 0)
+ %i.next = add i32 %i, 2
+ %cond = icmp ult i32 %i.next, 100
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ store <4 x float> %result, ptr addrspace(1) %out
+ ret void
+}
+
+declare <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half>, <16 x half>, <16 x half>, i1 immarg)
+declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg)
+declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16(<8 x half>, <16 x half>, <8 x float>, i16)
+declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half>, <8 x half>, <4 x float>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll b/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll
index 025008fac51441..ad3dbbd4b45122 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll
@@ -14,6 +14,7 @@
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -288,6 +289,7 @@
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll
index 2c31f5c9c34777..b2f5e154b4b99f 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll
@@ -13,6 +13,7 @@
; AFTER-PEI-NEXT: isChainFunction: false
; AFTER-PEI-NEXT: memoryBound: false
; AFTER-PEI-NEXT: waveLimiter: false
+; AFTER-PEI-NEXT: hasLDSContention: false
; AFTER-PEI-NEXT: hasSpilledSGPRs: true
; AFTER-PEI-NEXT: hasSpilledVGPRs: false
; AFTER-PEI-NEXT: hasNoWWMPoolSGPRSpillFallback: false
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
index 5497316fc19366..0196e7d9f89d72 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
@@ -14,6 +14,7 @@
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
index 8aab1c0fa55c8b..423083cbec37f5 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
@@ -14,6 +14,7 @@
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
index 9d1236b0f3739a..62db7f4108dc13 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
@@ -14,6 +14,7 @@
# FULL-NEXT: isChainFunction: false
# FULL-NEXT: memoryBound: true
# FULL-NEXT: waveLimiter: true
+# FULL-NEXT: hasLDSContention: false
# FULL-NEXT: hasSpilledSGPRs: false
# FULL-NEXT: hasSpilledVGPRs: false
# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -128,6 +129,7 @@ body: |
# FULL-NEXT: isChainFunction: false
# FULL-NEXT: memoryBound: false
# FULL-NEXT: waveLimiter: false
+# FULL-NEXT: hasLDSContention: false
# FULL-NEXT: hasSpilledSGPRs: false
# FULL-NEXT: hasSpilledVGPRs: false
# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -212,6 +214,7 @@ body: |
# FULL-NEXT: isChainFunction: false
# FULL-NEXT: memoryBound: false
# FULL-NEXT: waveLimiter: false
+# FULL-NEXT: hasLDSContention: false
# FULL-NEXT: hasSpilledSGPRs: false
# FULL-NEXT: hasSpilledVGPRs: false
# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -297,6 +300,7 @@ body: |
# FULL-NEXT: isChainFunction: false
# FULL-NEXT: memoryBound: false
# FULL-NEXT: waveLimiter: false
+# FULL-NEXT: hasLDSContention: false
# FULL-NEXT: hasSpilledSGPRs: false
# FULL-NEXT: hasSpilledVGPRs: false
# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
index 770718c166d4c1..9a42278cb9f884 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
@@ -17,6 +17,7 @@
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -81,6 +82,7 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -149,6 +151,7 @@ define amdgpu_ps void @gds_size_shader(i32 %arg0, i32 inreg %arg1) #5 {
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
@@ -209,6 +212,7 @@ define void @function() {
; CHECK-NEXT: isChainFunction: false
; CHECK-NEXT: memoryBound: false
; CHECK-NEXT: waveLimiter: false
+; CHECK-NEXT: hasLDSContention: false
; CHECK-NEXT: hasSpilledSGPRs: false
; CHECK-NEXT: hasSpilledVGPRs: false
; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false
More information about the llvm-branch-commits
mailing list