[llvm-branch-commits] [llvm] [AMDGPU] Add and plug-in DSLatencyMode flag + attr to control LDS latency (PR #205626)
Jeffrey Byrnes via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Jul 7 18:08:06 PDT 2026
https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/205626
>From c7221f445229a30f0ab02d1e5f4bc70242884341 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 23 Jun 2026 10:17:48 -0700
Subject: [PATCH 1/2] [AMDGPU] Add DSLatencyMode flag + attr to control LDS
latency
Change-Id: Ia5fee7983adffdb837ba2e876a64d805263b60c3
---
.../Target/AMDGPU/AMDGPUBarrierLatency.cpp | 5 +-
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 3 +-
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 23 ++--
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 3 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.cpp | 9 ++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 61 ++++++++-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 20 +++
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 4 +-
.../coexec-sched-flavor-classification.mir | 2 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 8 +-
.../CodeGen/AMDGPU/ds-latency-mode-attr.mir | 61 +++++++++
.../AMDGPU/ds-latency-mode-branch-cost.mir | 53 ++++++++
.../ds-latency-mode-default-scheduler.mir | 119 ++++++++++++++++++
.../CodeGen/AMDGPU/ds-latency-mode-flag.mir | 29 +++++
14 files changed, 373 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp b/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp
index 18b4727a8605b..d5f6b73753b1b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp
@@ -94,7 +94,6 @@ void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {
const unsigned BarrierSignalWaitLatency = BarrierSignalWaitLatencyOpt;
SmallVector<SUnit *, 8> RegionTDM;
SmallVector<SUnit *, 8> RegionAsync;
- const TargetSchedModel *SchedModel = DAG->getSchedModel();
for (SUnit &SU : DAG->SUnits) {
const MachineInstr *MI = SU.getInstr();
@@ -117,9 +116,9 @@ void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {
if (!MI->mayLoad() || MI->mayStore())
continue;
+ unsigned InstrLatency = TII->getInstrLatency(*MI);
addLatencyToEdge(PredDep, SU,
- SchedModel ? SchedModel->computeInstrLatency(MI, false)
- : FenceLatency);
+ InstrLatency ? InstrLatency : FenceLatency);
}
} else if (Op == AMDGPU::S_BARRIER_WAIT) {
for (SDep &PredDep : SU.Preds) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index d29ee5084cc6c..0daa1e1d88693 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -183,7 +183,7 @@ unsigned CandidateHeuristics::getHWUICyclesForInst(SUnit *SU) {
MachineInstr *MI = SU->getInstr();
if (MI->mayLoadOrStore())
- return SchedModel->computeInstrLatency(MI);
+ return SII->getInstrLatency(*MI);
unsigned ReleaseAtCycle = 0;
const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
@@ -714,6 +714,7 @@ ScheduleDAGInstrs *
llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
<< C->MF->getName() << '\n');
+
ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
C, std::make_unique<AMDGPUCoExecSchedStrategy>(C));
DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 722ddb5d0c4dd..7dc285a11ad1f 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -2188,7 +2188,7 @@ static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
unsigned Category = 0;
- unsigned Latency = SchedModel.computeInstrLatency(&MI);
+ unsigned Latency = TII->getInstrLatency(MI);
switch (Latency) {
case 8:
Category = IsSWMMAC ? 2 : 0;
@@ -2665,8 +2665,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
Register DstReg = MI.getOperand(0).getReg();
if (DstReg == Reg)
return false;
- HazardDefLatency =
- std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
+ HazardDefLatency = std::max(HazardDefLatency, TII.getInstrLatency(MI));
return TRI.regsOverlap(DstReg, Reg);
};
@@ -2742,8 +2741,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
if (!SIInstrInfo::isMFMA(MI))
return false;
Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
- HazardDefLatency =
- std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
+ HazardDefLatency = std::max(HazardDefLatency, TII.getInstrLatency(MI));
return TRI.regsOverlap(Reg, DstReg);
};
@@ -2904,8 +2902,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
(Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
- else if (ST.hasGFX940Insts() &&
- TSchedModel.computeInstrLatency(MI1) == 2)
+ else if (ST.hasGFX940Insts() && TII.getInstrLatency(*MI1) == 2)
NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
} else {
switch (Opc1) {
@@ -2925,7 +2922,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
break;
default:
- int NumPasses = TSchedModel.computeInstrLatency(MI1);
+ int NumPasses = TII.getInstrLatency(*MI1);
if (ST.hasGFX940Insts()) {
if (TII.isXDL(*MI) && !TII.isXDL(*MI1))
break;
@@ -2982,7 +2979,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
break;
default:
- int NumPasses = TSchedModel.computeInstrLatency(MI1);
+ int NumPasses = TII.getInstrLatency(*MI1);
if (ST.hasGFX940Insts()) {
NeedWaitStates =
@@ -3262,7 +3259,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
if (!MFMA)
continue;
- unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
+ unsigned HazardDefLatency = TII.getInstrLatency(*MFMA);
int NumPasses = HazardDefLatency;
int NeedWaitStates = MaxWaitStates;
@@ -3357,7 +3354,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
if (MFMA) {
int NeedWaitStates = MaxWaitStates;
- int NumPasses = TSchedModel.computeInstrLatency(MFMA);
+ int NumPasses = TII.getInstrLatency(*MFMA);
if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
switch (NumPasses) {
@@ -3424,7 +3421,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
if (!MFMA)
continue;
- unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
+ unsigned HazardDefLatency = TII.getInstrLatency(*MFMA);
int NeedWaitStates = MaxWaitStates;
switch (HazardDefLatency) {
case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
@@ -3463,7 +3460,7 @@ bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
if (IsMFMAFn(*MI)) {
int W = getWaitStatesSince(IsMFMAFn, 16);
if (MAI)
- return W < (int)TSchedModel.computeInstrLatency(MAI);
+ return W < (int)TII.getInstrLatency(*MAI);
}
return false;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index a4f854beaeebe..a59f6c643fca7 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1999,10 +1999,11 @@ GCNSchedStage::computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle,
DenseMap<unsigned, unsigned> &ReadyCycles,
const TargetSchedModel &SM) {
unsigned ReadyCycle = CurrCycle;
+ const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG.TII);
for (auto &D : SU.Preds) {
if (D.isAssignedRegDep()) {
MachineInstr *DefMI = D.getSUnit()->getInstr();
- unsigned Latency = SM.computeInstrLatency(DefMI);
+ unsigned Latency = SII->getInstrLatency(*DefMI);
unsigned DefReady = ReadyCycles[DAG.getSUnit(DefMI)->NodeNum];
ReadyCycle = std::max(ReadyCycle, DefReady + Latency);
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 37efb3a51cb9d..c2267780a84fc 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -786,6 +786,15 @@ void GCNSubtarget::adjustSchedDependency(
return; // This is not a data dependency anymore.
}
+ // DS load/store latency is variable depending on LDS contention.
+ if (InstrInfo.isDS(*DefI) &&
+ InstrInfo.getDSLatencyMultiplier(*DefI->getMF()) != 1) {
+ // For LDS instructions, we have overrides to change default latencies.
+ unsigned Latency = InstrInfo.getInstrLatency(*DefI);
+ Dep.setLatency(Latency);
+ return;
+ }
+
if (DefI->isBundle()) {
const SIRegisterInfo *TRI = getRegisterInfo();
auto Reg = Dep.getReg();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 43cdaa34cf3e3..c4e0c0d467541 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
+#include "AMDGPUTargetMachine.h"
#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
@@ -63,6 +64,17 @@ static cl::opt<bool> Fix16BitCopies(
cl::init(true),
cl::ReallyHidden);
+static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency(
+ "amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"),
+ cl::values(
+ clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast",
+ "Use default/pinned latency (no contention)"),
+ clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded",
+ "Use loaded latency (moderate contention, 3x latency)"),
+ clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded",
+ "Use overloaded latency (high contention, 5x latency)")),
+ cl::init(SIInstrInfo::DSLatencyMode::Fast), cl::Hidden);
+
SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
: AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
AMDGPU::ADJCALLSTACKDOWN),
@@ -10834,12 +10846,24 @@ unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
unsigned Lat = 0, Count = 0;
for (++I; I != E && I->isBundledWithPred(); ++I) {
++Count;
- Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I));
+ Lat = std::max(Lat, getInstrLatency(*I));
}
return Lat + Count - 1;
}
- return SchedModel.computeInstrLatency(&MI);
+ return getInstrLatency(MI);
+}
+
+unsigned SIInstrInfo::getInstrLatency(const MachineInstr &MI) const {
+ if (SchedModel.hasInstrSchedModel()) {
+ unsigned Latency = SchedModel.computeInstrLatency(&MI);
+ if (isDS(MI)) {
+ Latency *= getDSLatencyMultiplier(*MI.getMF());
+ }
+ return Latency;
+ }
+
+ return 0;
}
const MachineOperand &
@@ -11528,3 +11552,36 @@ bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
return AMDGPU::getMAIIsGFX940XDL(Opcode);
}
+
+unsigned SIInstrInfo::getDSLatencyMultiplier(const MachineFunction &MF) {
+ const Function &F = MF.getFunction();
+
+ // Priority selection goes to the attribute
+ Attribute A = F.getFnAttribute("amdgpu-ds-latency-mode");
+ if (A.isValid()) {
+ StringRef Val = A.getValueAsString();
+ if (Val == "fast")
+ return 1;
+ if (Val == "loaded")
+ return 3;
+ if (Val == "overloaded")
+ return 5;
+ }
+
+ // If using coexec scheduler, default to "loaded" mode unless overridden
+ // by the command line option.
+ if (DSLatency.getNumOccurrences() == 0 &&
+ AMDGPU::getSchedStrategy(F) == "coexec")
+ return 3;
+
+ switch (DSLatency) {
+ case DSLatencyMode::Fast:
+ return 1; // Use default scheduling model latency
+ case DSLatencyMode::Loaded:
+ return 3;
+ case DSLatencyMode::Overloaded:
+ return 5;
+ }
+
+ return 1;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 1d67c8664ff44..2359a7622b106 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1197,6 +1197,21 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
}
+ /// DS latency modes. The latency of DS load/store instructions
+ /// is variable depending on LDS contention.
+ enum class DSLatencyMode {
+ Fast, ///< Use default/pinned latency (no contention)
+ Loaded, ///< Use loaded latency (moderate contention, 3x latency)
+ Overloaded ///< Use overloaded latency (high contention, 5x latency)
+ };
+
+ /// \p returns the DS instruction latency multiplier based on the selected
+ /// DSLatencyMode. \p returns 1 if the default
+ /// scheduling model latency should be used (fast mode).
+ /// Checks the function attribute first, then if using coexec scheduler
+ /// defaults to "loaded", then falls back to the global command line option.
+ static unsigned getDSLatencyMultiplier(const MachineFunction &MF);
+
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
switch (Opcode) {
case AMDGPU::S_WAITCNT_soft:
@@ -1738,10 +1753,15 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
LiveIntervals *LIS = nullptr,
VirtRegMap *VRM = nullptr) const override;
+ // Silence a hidden overloaded virtual function warning.
+ using TargetInstrInfo::getInstrLatency;
+
unsigned getInstrLatency(const InstrItineraryData *ItinData,
const MachineInstr &MI,
unsigned *PredCost = nullptr) const override;
+ unsigned getInstrLatency(const MachineInstr &MI) const;
+
const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index a496c9a4daa71..19097118df3f0 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -421,14 +421,14 @@ class BranchWeightCostModel {
BranchProb = Head.getSuccProbability(FromIt);
if (BranchProb.isUnknown())
BranchProb = BranchProbability::getZero();
- BranchTakenCost = SchedModel.computeInstrLatency(&Branch);
+ BranchTakenCost = TII.getInstrLatency(Branch);
}
bool isProfitable(const MachineInstr &MI) {
if (TII.isWaitcnt(MI.getOpcode()))
return false;
- ThenCyclesCost += SchedModel.computeInstrLatency(&MI);
+ ThenCyclesCost += TII.getInstrLatency(MI);
// Consider `P = N/D` to be the probability of execz being false (skipping
// the then-block) The transformation is profitable if always executing the
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
index 82dd1d8748675..497558cab6e42 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
@@ -5,7 +5,7 @@
# CHECK-DAG: VALU(1c): 4 cycles, 4 instrs
# CHECK-DAG: TRANS: 2 cycles, 2 instrs
# CHECK-DAG: VMEM: 3200 cycles, 10 instrs
-# CHECK-DAG: DS: 80 cycles, 4 instrs
+# CHECK-DAG: DS: 240 cycles, 4 instrs
# CHECK-DAG: DMA: 640 cycles, 2 instrs
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index b225d8f18805a..61bfd9f0db77e 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -311,8 +311,6 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
-; COEXEC-NEXT: s_wait_dscnt 0x13
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
@@ -320,10 +318,12 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
-; COEXEC-NEXT: s_wait_dscnt 0x16
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
+; COEXEC-NEXT: s_wait_dscnt 0x1c
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; COEXEC-NEXT: s_wait_dscnt 0x18
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x14
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x10
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
new file mode 100644
index 0000000000000..59bc0c0554c9c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir
@@ -0,0 +1,61 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler %s -filetype=null 2>&1 | FileCheck %s
+
+
+# CHECK: Region: test_ds_latency_fast
+# CHECK: DS: 80 cycles, 4 instrs
+
+# CHECK: Region: test_ds_latency_loaded
+# CHECK: DS: 240 cycles, 4 instrs
+
+# CHECK: Region: test_ds_latency_overloaded
+# CHECK: DS: 400 cycles, 4 instrs
+
+--- |
+ define void @test_ds_latency_fast() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="fast" { ret void }
+ define void @test_ds_latency_loaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="loaded" { ret void }
+ define void @test_ds_latency_overloaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="overloaded" { ret void }
+
+...
+
+---
+name: test_ds_latency_fast
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
+
+---
+name: test_ds_latency_loaded
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
+
+---
+name: test_ds_latency_overloaded
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
new file mode 100644
index 0000000000000..95bf9df77965a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir
@@ -0,0 +1,53 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s
+
+# Test that ds-latency-mode affects the branch-over-removal cost model in
+# SIPreEmitPeephole. With higher DS latency, the cost of the then-block
+# is higher, making branch removal less profitable.
+
+---
+name: skip_execz_ds_multi
+body: |
+ ; FAST-LABEL: name: skip_execz_ds_multi
+ ; FAST: bb.0:
+ ; FAST-NEXT: successors: %bb.1(0x78000000)
+ ; FAST-NEXT: {{ $}}
+ ; FAST-NEXT: bb.1:
+ ; FAST-NEXT: successors: %bb.2(0x80000000)
+ ; FAST-NEXT: {{ $}}
+ ; FAST-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; FAST-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; FAST-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec
+ ; FAST-NEXT: {{ $}}
+ ; FAST-NEXT: bb.2:
+ ; FAST-NEXT: S_ENDPGM 0
+ ;
+ ; OVERLOADED-LABEL: name: skip_execz_ds_multi
+ ; OVERLOADED: bb.0:
+ ; OVERLOADED-NEXT: successors: %bb.1(0x78000000), %bb.2(0x08000000)
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: bb.1:
+ ; OVERLOADED-NEXT: successors: %bb.2(0x80000000)
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; OVERLOADED-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; OVERLOADED-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec
+ ; OVERLOADED-NEXT: {{ $}}
+ ; OVERLOADED-NEXT: bb.2:
+ ; OVERLOADED-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x78000000), %bb.2(0x08000000)
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+ bb.1:
+ successors: %bb.2
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
new file mode 100644
index 0000000000000..41695b863b00a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir
@@ -0,0 +1,119 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s
+
+---
+name: test_ds_latency_default_sched
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; FAST-LABEL: name: test_ds_latency_default_sched
+ ; FAST: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; FAST-NEXT: $m0 = S_MOV_B32 -1
+ ; FAST-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[DEF]], 0, 0, implicit $m0, implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DS_READ_B32_]], [[DS_READ_B32_]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_24:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_28:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_29:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: dead [[V_ADD_U32_e32_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; FAST-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_4]]
+ ;
+ ; OVERLOADED-LABEL: name: test_ds_latency_default_sched
+ ; OVERLOADED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; OVERLOADED-NEXT: $m0 = S_MOV_B32 -1
+ ; OVERLOADED-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[DEF]], 0, 0, implicit $m0, implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: [[V_ADD_U32_e32_24:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DS_READ_B32_]], [[DS_READ_B32_]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_28:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_29:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec
+ ; OVERLOADED-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_24]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ $m0 = S_MOV_B32 -1
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+ %3:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %4:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %5:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %6:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %15:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %16:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %17:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %18:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %19:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %20:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %21:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %22:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %23:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %24:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %25:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %26:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %27:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %28:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %29:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %30:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %31:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ %32:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec
+ S_ENDPGM 0, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
new file mode 100644
index 0000000000000..527390abcc8f5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir
@@ -0,0 +1,29 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=fast %s -filetype=null 2>&1 | FileCheck -check-prefix=FAST %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=loaded %s -filetype=null 2>&1 | FileCheck -check-prefix=LOADED %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -filetype=null 2>&1 | FileCheck -check-prefix=OVERLOADED %s
+
+# Test that the -amdgpu-ds-latency-mode command line flag affects DS cycle
+# accounting in the coexec scheduler.
+# DS instructions have base latency of 20 cycles each (4 instructions = 80 cycles).
+# - fast: 4 * 20 * 1 = 80 cycles
+# - loaded: 4 * 20 * 3 = 240 cycles
+# - overloaded: 4 * 20 * 5 = 400 cycles
+
+# FAST: DS: 80 cycles, 4 instrs
+# LOADED: DS: 240 cycles, 4 instrs
+# OVERLOADED: DS: 400 cycles, 4 instrs
+
+---
+name: test_ds_latency_flag
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec
+ %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec
+ DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+...
>From 3f913cfe6ee32823b1eddfffbfe904382468561c Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Tue, 7 Jul 2026 18:03:38 -0700
Subject: [PATCH 2/2] More documentation
Change-Id: I65ff1a1b605796d21a52766dcf04c64bc64aaa4e
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 27 ++++++++++++++++++++------
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c4e0c0d467541..18101bb70abb5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -68,7 +68,7 @@ static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency(
"amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"),
cl::values(
clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast",
- "Use default/pinned latency (no contention)"),
+ "Use default/pinged latency (no contention)"),
clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded",
"Use loaded latency (moderate contention, 3x latency)"),
clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded",
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2359a7622b106..a513ecf8652a7 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1197,17 +1197,29 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
}
- /// DS latency modes. The latency of DS load/store instructions
- /// is variable depending on LDS contention.
+ /// DS latency modes. The latency of an individual DS load/store instruction
+ /// is variable. Some of the major sources that cause this variation are hard
+ /// to model at compile time. For example, if we have multiple LDS
+ /// instructions in flight, one of these may take longer than the other due to
+ /// a bank conflict. Given that these bank conflicts can occur across waves,
+ /// it is hard to accurately model this. This is compounded given the presence
+ /// of the LDS FIFO -- if an earlier LDS instruction has, for example, a bank
+ /// conflict, this will impact the latency of the current LDS instruction.
+ ///
+ /// Given these complexities, we offer different DSLatencyModes for kernels to
+ /// express the average latency for LDS instructions in the kernel. We expect
+ /// that for kernels with many closely packed LDS isntructions, the average
+ /// latency for LDS instructions will be relatively high. Thus we should use
+ /// DSLatencyMode::Loaded or DSLatencyMode::Overloaded.
enum class DSLatencyMode {
- Fast, ///< Use default/pinned latency (no contention)
- Loaded, ///< Use loaded latency (moderate contention, 3x latency)
- Overloaded ///< Use overloaded latency (high contention, 5x latency)
+ Fast, // Use default/pinged latency (no contention)
+ Loaded, // Use loaded latency (moderate contention, 3x latency)
+ Overloaded // Use overloaded latency (high contention, 5x latency)
};
/// \p returns the DS instruction latency multiplier based on the selected
/// DSLatencyMode. \p returns 1 if the default
- /// scheduling model latency should be used (fast mode).
+ /// scheduling model latency should be used (pinged mode).
/// Checks the function attribute first, then if using coexec scheduler
/// defaults to "loaded", then falls back to the global command line option.
static unsigned getDSLatencyMultiplier(const MachineFunction &MF);
@@ -1760,6 +1772,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
const MachineInstr &MI,
unsigned *PredCost = nullptr) const override;
+ /// \returns the latency of a given instruction \p MI. This implements custom
+ /// overrides for certain cases (e.g. when using dfifferent DSLatencyModes to
+ /// express increased LDS resource contention).
unsigned getInstrLatency(const MachineInstr &MI) const;
const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
More information about the llvm-branch-commits
mailing list