[llvm] [AMDGPU] Add stalls for DS FIFO buffer (PR #192323)
Austin Kerbow via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 17:49:42 PDT 2026
https://github.com/kerbowa updated https://github.com/llvm/llvm-project/pull/192323
>From 7fe0afd20065d5ce575fc5719eeaf6aeb8e812a1 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Thu, 28 May 2026 09:59:42 -0700
Subject: [PATCH 1/8] [AMDGPU] Add stalls for DS FIFO buffer
Change-Id: I73e56da97a931349e0655e4e20b24aeb97920647
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 46 ++++++++++++++++--
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 47 ++++++++++++++++---
2 files changed, 82 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 726be8c7f0982..790406a9904bd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -142,10 +142,12 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
if (TotalCycles == 0)
return;
+ ScheduledSUs.push_back(SU);
AllSUs.remove(SU);
PrioritySUs.remove(SU);
- TotalCycles -= BlockingCycles;
+ if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
+ TotalCycles -= BlockingCycles;
if (AllSUs.empty())
return;
@@ -172,6 +174,14 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
}
}
+void HardwareUnitInfo::finalizeCycles() {
+ if (BufferSize <= 1 || !AllSUs.size())
+ return;
+
+ BufferCycles = TotalCycles / AllSUs.size();
+ TotalCycles /= BufferSize;
+}
+
HardwareUnitInfo *
CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
for (HardwareUnitInfo &HWUICand : HWUInfo) {
@@ -221,6 +231,7 @@ void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
HWUInfo[(int)InstructionFlavor::WMMA].setProducesCoexecWindow(true);
HWUInfo[(int)InstructionFlavor::MultiCycleVALU].setProducesCoexecWindow(true);
HWUInfo[(int)InstructionFlavor::TRANS].setProducesCoexecWindow(true);
+ HWUInfo[(int)InstructionFlavor::DS].setBufferSize(DefaultBufferSizes::DS);
collectHWUIPressure();
}
@@ -234,6 +245,10 @@ void CandidateHeuristics::collectHWUIPressure() {
HWUInfo[(int)(Flavor)].insert(&SU, getHWUICyclesForInst(&SU));
}
+ for (auto &HWUI : HWUInfo) {
+ HWUI.finalizeCycles();
+ }
+
LLVM_DEBUG(dumpRegionSummary());
}
@@ -681,7 +696,26 @@ bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
SchedCandidate &TryCand,
- SchedBoundary &Zone) const {
+ SchedBoundary &Zone) {
+ auto getBufferFullStalls = [this,
+ &Zone](SUnit *SU) -> unsigned {
+ InstructionFlavor Flavor = classifyFlavor(
+ *SU->getInstr(), *static_cast<const SIInstrInfo *>(DAG->TII));
+ HardwareUnitInfo *HWUI = Heurs.getHWUIFromFlavor(Flavor);
+
+ if (HWUI->getBufferSize() <= 1)
+ return 0;
+
+ // getBufferAvailableCycle assumes top-down scheduling.
+ assert(Zone.isTop());
+ unsigned CurrCycle = Zone.getCurrCycle();
+ unsigned BufferReadyCycle = HWUI->getBufferAvailableCycle(CurrCycle);
+ if (BufferReadyCycle <= CurrCycle)
+ return 0;
+
+ return BufferReadyCycle - CurrCycle;
+ };
+
// Treat structural and latency stalls as a single scheduling cost for the
// current cycle.
struct StallCosts {
@@ -689,6 +723,7 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
unsigned Structural = 0;
unsigned Latency = 0;
unsigned Effective = 0;
+ unsigned Buffer = 0;
};
unsigned CurrCycle = Zone.getCurrCycle();
@@ -698,7 +733,8 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
Costs.Structural = getStructuralStallCycles(Zone, SU);
Costs.Latency = Zone.getLatencyStallCycles(SU);
- Costs.Effective = std::max({Costs.Ready, Costs.Structural, Costs.Latency});
+ Costs.Buffer = getBufferFullStalls(SU);
+ Costs.Effective = std::max({Costs.Ready, Costs.Structural, Costs.Latency, Costs.Buffer});
return Costs;
};
@@ -708,10 +744,10 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
dbgs() << "Effective stalls: try=" << TryCosts.Effective
<< " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
- << ", lat=" << TryCosts.Latency << ") cand=" << CandCosts.Effective
+ << ", lat=" << TryCosts.Latency << ", buffer=" << TryCosts.Buffer << ") cand=" << CandCosts.Effective
<< " (ready=" << CandCosts.Ready
<< ", struct=" << CandCosts.Structural
- << ", lat=" << CandCosts.Latency << ")\n";
+ << ", lat=" << CandCosts.Latency << ", buffer=" << CandCosts.Buffer << ")\n";
});
return tryLess(TryCosts.Effective, CandCosts.Effective, TryCand, Cand, Stall);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 4ea6853e46a64..bff84e2920d86 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -21,6 +21,9 @@
namespace llvm {
namespace AMDGPU {
+namespace DefaultBufferSizes {
+constexpr unsigned DS = 16;
+} // namespace DefaultBufferSizes
/// AMDGPU-specific scheduling decision reasons. These provide more granularity
/// than the generic CandReason enum for debugging purposes.
@@ -66,6 +69,8 @@ class HardwareUnitInfo {
SmallSetVector<SUnit *, 16> PrioritySUs;
/// All the SUs in the region that consume this resource.
SmallSetVector<SUnit *, 16> AllSUs;
+ /// All the SUs for this HardwareUnit that have already been scheduled.
+ SmallVector<SUnit *, 16> ScheduledSUs;
/// The total number of busy cycles for this HardwareUnit for a given region.
unsigned TotalCycles = 0;
/// InstructionFlavor mapping.
@@ -75,6 +80,11 @@ class HardwareUnitInfo {
/// / MFMA instructions may take multiple cycles, which may be overlapped with
/// instructions on other HardwareUnits.
bool ProducesCoexecWindow = false;
+ /// How many instructons can be held simultaneously for this HardwareUnit.
+ /// A value of 0 or 1 means that there is no buffer.
+ unsigned BufferSize = 0;
+ /// How many cycles it takes for an instruction to clear the buffer.
+ unsigned BufferCycles = 0;
public:
HardwareUnitInfo() {}
@@ -96,6 +106,24 @@ class HardwareUnitInfo {
bool contains(SUnit *SU) const { return AllSUs.contains(SU); }
+ void setBufferSize(unsigned Size) { BufferSize = Size; }
+
+ unsigned getBufferSize() { return BufferSize; }
+
+ /// \returns the next cycle where there is space in the buffer.
+ unsigned getBufferAvailableCycle(unsigned CurrCycle) {
+ // There is no buffer.
+ if (BufferSize <= 1)
+ return CurrCycle;
+
+ // Buffer is available now.
+ if (ScheduledSUs.size() < BufferSize)
+ return CurrCycle;
+
+ return BufferCycles +
+ ScheduledSUs[ScheduledSUs.size() - BufferSize]->TopReadyCycle;
+ }
+
/// \returns the SUnit with higher priority or nullptr if they are the same.
/// This method looks through the PrioritySUs to determine if one SU is more
/// prioritized than the other. If neither are in the PrioritySUs list, then
@@ -117,6 +145,8 @@ class HardwareUnitInfo {
TotalCycles = 0;
Type = AMDGPU::InstructionFlavor::Other;
ProducesCoexecWindow = false;
+ BufferSize = 0;
+ BufferCycles = 0;
}
/// \returns the next SU in PrioritySUs that is not ready. If \p LookDeep is
@@ -136,6 +166,11 @@ class HardwareUnitInfo {
/// and reducing its \p BlockingCycles from the TotalCycles. This maintains
/// the list of PrioritySUs.
void markScheduled(SUnit *SU, unsigned BlockingCycles);
+ /// After we've collected all the region pressure for this HWUI, correct for
+ /// any specifics of the behavior of this resource. For example, if we the
+ /// HardwareUnit can hold N instructions simultaneously, then there is no
+ /// penalty for scheduling N instructions back to back.
+ void finalizeCycles();
};
//===----------------------------------------------------------------------===//
@@ -160,10 +195,6 @@ class CandidateHeuristics {
/// SU.
unsigned getHWUICyclesForInst(SUnit *SU);
- /// Given a \p Flavor , find the corresponding HardwareUnit. \returns the
- /// mapped HardwareUnit.
- HardwareUnitInfo *getHWUIFromFlavor(AMDGPU::InstructionFlavor Flavor);
-
public:
CandidateHeuristics() = default;
@@ -173,7 +204,11 @@ class CandidateHeuristics {
/// Update the state to reflect that \p SU is going to be scheduled.
void updateForScheduling(SUnit *SU);
- /// Sort the HWUInfo vector. After sorting, the HardwareUnits that are highest
+ /// Given a \p Flavor , find the corresponding HardwareUnit. \returns the
+ /// mapped HardwareUnit.
+ HardwareUnitInfo *getHWUIFromFlavor(AMDGPU::InstructionFlavor Flavor);
+
+ /// Sort the HardwarUnitInfo vector. After sorting, the HWUI that are highest
/// priority are first. Priority is determined by maximizing coexecution and
/// keeping the critical HardwareUnit busy.
void sortHWUIResources();
@@ -202,7 +237,7 @@ class CandidateHeuristics {
class AMDGPUCoExecSchedStrategy final : public GCNSchedStrategy {
protected:
bool tryEffectiveStall(SchedCandidate &Cand, SchedCandidate &TryCand,
- SchedBoundary &Zone) const;
+ SchedBoundary &Zone);
AMDGPU::AMDGPUSchedReason LastAMDGPUReason = AMDGPU::AMDGPUSchedReason::None;
CandidateHeuristics Heurs;
>From 80c4762d3860db6e638efd41b9d100dfe14d525d Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 13 Mar 2026 14:29:33 -0700
Subject: [PATCH 2/8] Typo
Change-Id: I8b8da8a07be84506483f474d0a5e10ad79178c15
---
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index bff84e2920d86..8d1d16850a61b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -80,7 +80,7 @@ class HardwareUnitInfo {
/// / MFMA instructions may take multiple cycles, which may be overlapped with
/// instructions on other HardwareUnits.
bool ProducesCoexecWindow = false;
- /// How many instructons can be held simultaneously for this HardwareUnit.
+ /// How many instructions can be held simultaneously for this HardwareUnit.
/// A value of 0 or 1 means that there is no buffer.
unsigned BufferSize = 0;
/// How many cycles it takes for an instruction to clear the buffer.
>From 82ef03fc9729c805fd425d93418bd31fc8ee8940 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 23 Mar 2026 15:37:51 -0700
Subject: [PATCH 3/8] Merge conflicts
Change-Id: I33564a1e5d14f3b53577cb463ba2cb3a7993fd24
---
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 790406a9904bd..838446304c345 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -744,10 +744,11 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
dbgs() << "Effective stalls: try=" << TryCosts.Effective
<< " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
- << ", lat=" << TryCosts.Latency << ", buffer=" << TryCosts.Buffer << ") cand=" << CandCosts.Effective
- << " (ready=" << CandCosts.Ready
+ << ", lat=" << TryCosts.Latency << ", buffer=" << TryCosts.Buffer
+ << ") cand=" << CandCosts.Effective << " (ready=" << CandCosts.Ready
<< ", struct=" << CandCosts.Structural
- << ", lat=" << CandCosts.Latency << ", buffer=" << CandCosts.Buffer << ")\n";
+ << ", lat=" << CandCosts.Latency << ", buffer=" << CandCosts.Buffer
+ << ")\n";
});
return tryLess(TryCosts.Effective, CandCosts.Effective, TryCand, Cand, Stall);
>From 29cb18da0ac3828ec0779932ef5a90d42761602c Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 23 Mar 2026 15:59:16 -0700
Subject: [PATCH 4/8] Claude Code review
Change-Id: Id4983ca59270c8bb2d261d38a6e7f2483c9d237e
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 22 +++++++++++++++----
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 1 +
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 838446304c345..88c6e20b53824 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -175,10 +175,24 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
}
void HardwareUnitInfo::finalizeCycles() {
- if (BufferSize <= 1 || !AllSUs.size())
+ if (BufferSize <= 1 || AllSUs.empty())
return;
+ // We estimate the amount of cycles it takes to free up a slot in the buffer
+ // as the average cycles per SU.
BufferCycles = TotalCycles / AllSUs.size();
+ // The TotalCycles is normalized against the BufferSize.
+ // This provides an estimate of the TotalCycles which is not always accurate
+ // -- particularly in cases where we have fewer instructions than the
+ // BufferSize. For example, if we have 2 instructions which each take 50
+ // cycles and a BufferSize of 16, then a TotalCycles of 51 cycles would be
+ // somewhat accurate. This normalization calculates TotalCycles as 6. However,
+ // if we have 64 of these instructions, our normalized estimate of 200 is more
+ // reasonable, given the more accurate measure is 264. Having a completely
+ // accurate measure is not very important, since this metric is mainly used to
+ // compare the relative demand per HardwareUnit across the region. The simpler
+ // estimate makes managing the metric incrementally during scheduling much
+ // simpler.
TotalCycles /= BufferSize;
}
@@ -697,8 +711,7 @@ bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
SchedCandidate &TryCand,
SchedBoundary &Zone) {
- auto getBufferFullStalls = [this,
- &Zone](SUnit *SU) -> unsigned {
+ auto getBufferFullStalls = [this, &Zone](SUnit *SU) -> unsigned {
InstructionFlavor Flavor = classifyFlavor(
*SU->getInstr(), *static_cast<const SIInstrInfo *>(DAG->TII));
HardwareUnitInfo *HWUI = Heurs.getHWUIFromFlavor(Flavor);
@@ -734,7 +747,8 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
Costs.Structural = getStructuralStallCycles(Zone, SU);
Costs.Latency = Zone.getLatencyStallCycles(SU);
Costs.Buffer = getBufferFullStalls(SU);
- Costs.Effective = std::max({Costs.Ready, Costs.Structural, Costs.Latency, Costs.Buffer});
+ Costs.Effective =
+ std::max({Costs.Ready, Costs.Structural, Costs.Latency, Costs.Buffer});
return Costs;
};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 8d1d16850a61b..ea98e56d3d880 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -142,6 +142,7 @@ class HardwareUnitInfo {
void reset() {
AllSUs.clear();
PrioritySUs.clear();
+ ScheduledSUs.clear();
TotalCycles = 0;
Type = AMDGPU::InstructionFlavor::Other;
ProducesCoexecWindow = false;
>From 8a8f9f616173e0c41fce9d4033a2fc2e2a304ed5 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Fri, 24 Apr 2026 09:29:10 -0700
Subject: [PATCH 5/8] Address Review comments
Change-Id: I6972e887edd5db44ee9bcaed1f79e0c9933f611e
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 7 ++++++-
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 20 ++++++++++++++++---
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 88c6e20b53824..5cd5625648d1f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -146,6 +146,8 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
AllSUs.remove(SU);
PrioritySUs.remove(SU);
+ // BufferSize of 0 or 1 implies that each SU uses the HardwareUnit for
+ // BlockingCycles
if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
TotalCycles -= BlockingCycles;
@@ -175,6 +177,8 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
}
void HardwareUnitInfo::finalizeCycles() {
+ // BufferSize of 0 or 1 implies that each SU uses the HardwareUnit for
+ // BlockingCycles
if (BufferSize <= 1 || AllSUs.empty())
return;
@@ -716,7 +720,8 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
*SU->getInstr(), *static_cast<const SIInstrInfo *>(DAG->TII));
HardwareUnitInfo *HWUI = Heurs.getHWUIFromFlavor(Flavor);
- if (HWUI->getBufferSize() <= 1)
+ // A BufferSize of 0 means "unlimited" buffer, thus we will never fill it.
+ if (HWUI->getBufferSize() == 0)
return 0;
// getBufferAvailableCycle assumes top-down scheduling.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index ea98e56d3d880..b7ffa52a09891 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -81,9 +81,23 @@ class HardwareUnitInfo {
/// instructions on other HardwareUnits.
bool ProducesCoexecWindow = false;
/// How many instructions can be held simultaneously for this HardwareUnit.
- /// A value of 0 or 1 means that there is no buffer.
+ /// A value of 0 means there is no limit.
+ ///
+ /// This may approximate the hardware. For example, for LDS instructions
+ /// it is a well-known phenomena that oversubscribing the LDS unit results in
+ /// longer latency for the LDS instructions. While it is true that there is a
+ /// hard limit to the amount of simulatenous in-flight LDS instructions, good
+ /// scheduling would also cool off the LDS to avoid other forms of hardware
+ /// contention and increasing LDS latency. Thus, we limit the amount of LDS
+ /// instructions we are willing to schedule close together, though this does
+ /// not correspond 1:1 with a hardware mechanism.
unsigned BufferSize = 0;
/// How many cycles it takes for an instruction to clear the buffer.
+ ///
+ /// Again, this may be an apprxoimation. For example, for memory FIFOs, the
+ /// actual amount of cycles it will take to clear it is dependent on how
+ /// quickly prior instructions evacuate the FIFO, which is based on runtime
+ /// behavior which is not modelled in the compiler.
unsigned BufferCycles = 0;
public:
@@ -113,7 +127,7 @@ class HardwareUnitInfo {
/// \returns the next cycle where there is space in the buffer.
unsigned getBufferAvailableCycle(unsigned CurrCycle) {
// There is no buffer.
- if (BufferSize <= 1)
+ if (BufferSize == 0)
return CurrCycle;
// Buffer is available now.
@@ -168,7 +182,7 @@ class HardwareUnitInfo {
/// the list of PrioritySUs.
void markScheduled(SUnit *SU, unsigned BlockingCycles);
/// After we've collected all the region pressure for this HWUI, correct for
- /// any specifics of the behavior of this resource. For example, if we the
+ /// any specifics of the behavior of this resource. For example, if the
/// HardwareUnit can hold N instructions simultaneously, then there is no
/// penalty for scheduling N instructions back to back.
void finalizeCycles();
>From 7c13a9280f3f639c1257d259eee4a8e472a571da Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 15 Jun 2026 13:44:10 -0700
Subject: [PATCH 6/8] [AMDGPU] Use DS latency for FIFO scheduling
Use instruction latency for DS hardware-unit cycle accounting so the FIFO
model can identify a full buffer. Add focused MIR coverage for the resulting
stall cost and scheduling decision, and regenerate the integration checks.
Change-Id: I2f4df2e97d145af4935872dbd43108e1b55077ab
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 4 ++
.../CodeGen/AMDGPU/coexec-hazardrec-preRA.mir | 10 ++--
.../CodeGen/AMDGPU/coexec-sched-ds-fifo.mir | 48 +++++++++++++++++++
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 8 ++--
4 files changed, 61 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 5cd5625648d1f..1661dd201ba3a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -212,6 +212,10 @@ CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
unsigned CandidateHeuristics::getHWUICyclesForInst(SUnit *SU) {
assert(SchedModel && SchedModel->hasInstrSchedModel());
+ MachineInstr *MI = SU->getInstr();
+ if (SII->isDS(*MI))
+ return SchedModel->computeInstrLatency(MI);
+
unsigned ReleaseAtCycle = 0;
const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
for (TargetSchedModel::ProcResIter PI = SchedModel->getWriteProcResBegin(SC),
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir b/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
index b4e8a86de06b8..43a735248ea5a 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
@@ -35,7 +35,7 @@
# DBG: CoExec window complete:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
-# DBG-NEXT: Scheduled: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 E E I E E I - V -
---
name: wmma_ds_salu_only
tracksRegLiveness: true
@@ -58,12 +58,12 @@ body: |
; CHECK-NEXT: early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF6]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_1]], [[DEF6]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 64, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF6]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_1]], [[DEF6]], implicit-def dead $scc
; CHECK-NEXT: [[S_ADD_I32_3:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_2]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[DS_READ_B128_gfx9_4]], implicit [[S_ADD_I32_3]], implicit %17
bb.0:
@@ -340,8 +340,8 @@ body: |
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF10]], [[DEF11]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF10]], [[DEF11]], implicit-def dead $scc
; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF8]], implicit $mode, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF10]], implicit-def dead $scc
; CHECK-NEXT: [[V_EXP_F32_e32_1:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF9]], implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
new file mode 100644
index 0000000000000..05a08205ea767
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
@@ -0,0 +1,48 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler %s -filetype=null 2>&1 | FileCheck --check-prefix=DEBUG %s
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck --check-prefix=ORDER %s
+
+# DEBUG: Effective stalls: try=4 (ready=0, struct=0, lat=0, buffer=4) cand=0 (ready=0, struct=0, lat=0, buffer=0)
+
+# ORDER-LABEL: name: ds_fifo_stall
+# ORDER: body: |
+# ORDER-NEXT: bb.0:
+# ORDER-NEXT: %0:vgpr_32 = IMPLICIT_DEF
+# ORDER-COUNT-16: DS_READ_B32
+# ORDER-NEXT: %18:vgpr_32 = V_MOV_B32_e32 %0, implicit $exec
+# ORDER-NEXT: %17:vgpr_32 = DS_READ_B32
+
+--- |
+ @lds = internal addrspace(3) global [17 x i32] undef
+
+ define void @ds_fifo_stall() #0 { ret void }
+
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
+...
+
+---
+name: ds_fifo_stall
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 0)`, addrspace 3)
+ %2:vgpr_32 = DS_READ_B32 %0, 4, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 1)`, addrspace 3)
+ %3:vgpr_32 = DS_READ_B32 %0, 8, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 2)`, addrspace 3)
+ %4:vgpr_32 = DS_READ_B32 %0, 12, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 3)`, addrspace 3)
+ %5:vgpr_32 = DS_READ_B32 %0, 16, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 4)`, addrspace 3)
+ %6:vgpr_32 = DS_READ_B32 %0, 20, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 5)`, addrspace 3)
+ %7:vgpr_32 = DS_READ_B32 %0, 24, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 6)`, addrspace 3)
+ %8:vgpr_32 = DS_READ_B32 %0, 28, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 7)`, addrspace 3)
+ %9:vgpr_32 = DS_READ_B32 %0, 32, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 8)`, addrspace 3)
+ %10:vgpr_32 = DS_READ_B32 %0, 36, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 9)`, addrspace 3)
+ %11:vgpr_32 = DS_READ_B32 %0, 40, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 10)`, addrspace 3)
+ %12:vgpr_32 = DS_READ_B32 %0, 44, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 11)`, addrspace 3)
+ %13:vgpr_32 = DS_READ_B32 %0, 48, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 12)`, addrspace 3)
+ %14:vgpr_32 = DS_READ_B32 %0, 52, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 13)`, addrspace 3)
+ %15:vgpr_32 = DS_READ_B32 %0, 56, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 14)`, addrspace 3)
+ %16:vgpr_32 = DS_READ_B32 %0, 60, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 15)`, addrspace 3)
+ %17:vgpr_32 = DS_READ_B32 %0, 64, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 16)`, addrspace 3)
+ %18:vgpr_32 = V_MOV_B32_e32 %0, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18
+...
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index b1d2d54c5d350..ceb36b36bffee 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -315,21 +315,21 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v124 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v124 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v124 offset:384
+; COEXEC-NEXT: s_wait_dscnt 0xf
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v124 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
-; COEXEC-NEXT: s_wait_dscnt 0x13
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
+; COEXEC-NEXT: s_wait_dscnt 0x12
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
-; COEXEC-NEXT: s_wait_dscnt 0x16
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
; COEXEC-NEXT: s_wait_dscnt 0x14
>From fd19af16892afaeebd3214972816cda4e61e0cc8 Mon Sep 17 00:00:00 2001
From: Austin Kerbow <Austin.Kerbow at amd.com>
Date: Tue, 18 Aug 2026 19:17:45 -0500
Subject: [PATCH 7/8] Handle DS FIFO accounting edge cases
Saturate hardware-unit pressure decrements and treat buffer sizes zero
and one as disabling buffering to avoid underflow and inconsistent stall
costs.
---
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 9 ++++-----
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 4 ++--
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 1661dd201ba3a..715b8dc07799b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -149,7 +149,7 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
// BufferSize of 0 or 1 implies that each SU uses the HardwareUnit for
// BlockingCycles
if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
- TotalCycles -= BlockingCycles;
+ TotalCycles -= std::min(TotalCycles, BlockingCycles);
if (AllSUs.empty())
return;
@@ -267,9 +267,8 @@ void CandidateHeuristics::collectHWUIPressure() {
HWUInfo[(int)(Flavor)].insert(&SU, getHWUICyclesForInst(&SU));
}
- for (auto &HWUI : HWUInfo) {
+ for (auto &HWUI : HWUInfo)
HWUI.finalizeCycles();
- }
LLVM_DEBUG(dumpRegionSummary());
}
@@ -724,8 +723,8 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
*SU->getInstr(), *static_cast<const SIInstrInfo *>(DAG->TII));
HardwareUnitInfo *HWUI = Heurs.getHWUIFromFlavor(Flavor);
- // A BufferSize of 0 means "unlimited" buffer, thus we will never fill it.
- if (HWUI->getBufferSize() == 0)
+ // A BufferSize of 0 or 1 means there is no buffered scheduling cost.
+ if (HWUI->getBufferSize() <= 1)
return 0;
// getBufferAvailableCycle assumes top-down scheduling.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index b7ffa52a09891..cdb0764eaf8d4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -81,7 +81,7 @@ class HardwareUnitInfo {
/// instructions on other HardwareUnits.
bool ProducesCoexecWindow = false;
/// How many instructions can be held simultaneously for this HardwareUnit.
- /// A value of 0 means there is no limit.
+ /// A value of 0 or 1 disables buffering.
///
/// This may approximate the hardware. For example, for LDS instructions
/// it is a well-known phenomena that oversubscribing the LDS unit results in
@@ -127,7 +127,7 @@ class HardwareUnitInfo {
/// \returns the next cycle where there is space in the buffer.
unsigned getBufferAvailableCycle(unsigned CurrCycle) {
// There is no buffer.
- if (BufferSize == 0)
+ if (BufferSize <= 1)
return CurrCycle;
// Buffer is available now.
>From 503bfb412d0e148a1531ffed2b9481e48eff5564 Mon Sep 17 00:00:00 2001
From: Austin Kerbow <Austin.Kerbow at amd.com>
Date: Tue, 18 Aug 2026 19:47:17 -0500
Subject: [PATCH 8/8] [AMDGPU] Correct DS FIFO buffer size semantics
There was some ambiguity in how buffersize 0 and 1 are handled. The
correct semantics are:
- `BufferSize == 0`: unlimited, no FIFO stall
- `BufferSize == 1`: unbuffered, only one instruction in flight
- `BufferSize > 1`: buffered FIFO
---
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 16 +++++++++-------
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 7 ++++---
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 715b8dc07799b..ccd10a0c9d004 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -146,8 +146,8 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
AllSUs.remove(SU);
PrioritySUs.remove(SU);
- // BufferSize of 0 or 1 implies that each SU uses the HardwareUnit for
- // BlockingCycles
+ // BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
+ // either case, each SU uses the HardwareUnit for BlockingCycles.
if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
TotalCycles -= std::min(TotalCycles, BlockingCycles);
@@ -177,14 +177,16 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
}
void HardwareUnitInfo::finalizeCycles() {
- // BufferSize of 0 or 1 implies that each SU uses the HardwareUnit for
- // BlockingCycles
- if (BufferSize <= 1 || AllSUs.empty())
+ if (BufferSize == 0 || AllSUs.empty())
return;
// We estimate the amount of cycles it takes to free up a slot in the buffer
// as the average cycles per SU.
BufferCycles = TotalCycles / AllSUs.size();
+ // A single-entry buffer does not reduce TotalCycles.
+ if (BufferSize == 1)
+ return;
+
// The TotalCycles is normalized against the BufferSize.
// This provides an estimate of the TotalCycles which is not always accurate
// -- particularly in cases where we have fewer instructions than the
@@ -723,8 +725,8 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
*SU->getInstr(), *static_cast<const SIInstrInfo *>(DAG->TII));
HardwareUnitInfo *HWUI = Heurs.getHWUIFromFlavor(Flavor);
- // A BufferSize of 0 or 1 means there is no buffered scheduling cost.
- if (HWUI->getBufferSize() <= 1)
+ // A BufferSize of 0 is unlimited, so it has no FIFO scheduling cost.
+ if (HWUI->getBufferSize() == 0)
return 0;
// getBufferAvailableCycle assumes top-down scheduling.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index cdb0764eaf8d4..672c5e3c33b97 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -81,7 +81,8 @@ class HardwareUnitInfo {
/// instructions on other HardwareUnits.
bool ProducesCoexecWindow = false;
/// How many instructions can be held simultaneously for this HardwareUnit.
- /// A value of 0 or 1 disables buffering.
+ /// A value of 0 means there is no limit. A value of 1 models an unbuffered
+ /// resource with a single in-flight instruction.
///
/// This may approximate the hardware. For example, for LDS instructions
/// it is a well-known phenomena that oversubscribing the LDS unit results in
@@ -126,8 +127,8 @@ class HardwareUnitInfo {
/// \returns the next cycle where there is space in the buffer.
unsigned getBufferAvailableCycle(unsigned CurrCycle) {
- // There is no buffer.
- if (BufferSize <= 1)
+ // An unlimited buffer is always available.
+ if (BufferSize == 0)
return CurrCycle;
// Buffer is available now.
More information about the llvm-commits
mailing list