[llvm] 3638e00 - [AMDGPU] Add stalls for DS FIFO buffer (#192323)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 23:53:25 PDT 2026
Author: Jeffrey Byrnes
Date: 2026-08-25T23:53:20-07:00
New Revision: 3638e00f5a7e6b57ff62ced62388a60011d1a8db
URL: https://github.com/llvm/llvm-project/commit/3638e00f5a7e6b57ff62ced62388a60011d1a8db
DIFF: https://github.com/llvm/llvm-project/commit/3638e00f5a7e6b57ff62ced62388a60011d1a8db.diff
LOG: [AMDGPU] Add stalls for DS FIFO buffer (#192323)
This PR adds modeling for the LDS FIFO, and adds support for modelling
other types of buffers with similar characteristics.
The LDS unit has a FIFO which allows us to issue multiple LDS
instructions while the first is still executing. However, the FIFO is
sized, and, if we exceed the capacity, we will need to wait for a spot
in the FIFO before we can issue the instruction. This adds modeling for
this feature, which has impacts on: 1. the stall cycle calculations, and
2. the total usage of the HardwareUnit. The reason why this impacts 2.
is because previously we just summed the cycle count for each LDS
instruction for the total cycle. But, we can actually run N LDS
instructions in N + SingleInstructionCycle * N / BufferSize cycles. So,
if we scheduled 2 DS_LOAD instructions back to back, with
SingleInstructionCycle of 20 and BufferSize of 16, it would take 21
cycles to execute them instead of 40.
The SchedModel doesn't really have any good way to express this. It has
BufferSize property for ProcResource, but this is mainly used for OOO
scheduling, and it doesn't have any impact on stalling behavior. We can
replicate the LDS ProcResource N times, but this is hacking the
SchedModel to get the desired stall behavior while misrepresenting the
hardware. In this modelling, if we ever need to understand that two DS
instructions actually use the same ProcResource, we will need to write
custom C++ on top of the SchedModel.
---------
Co-authored-by: Austin Kerbow <Austin.Kerbow at amd.com>
Added:
llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
Modified:
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 726be8c7f0982..ccd10a0c9d004 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -142,10 +142,14 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
if (TotalCycles == 0)
return;
+ ScheduledSUs.push_back(SU);
AllSUs.remove(SU);
PrioritySUs.remove(SU);
- TotalCycles -= BlockingCycles;
+ // BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
+ // either case, each SU uses the HardwareUnit for BlockingCycles.
+ if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
+ TotalCycles -= std::min(TotalCycles, BlockingCycles);
if (AllSUs.empty())
return;
@@ -172,6 +176,32 @@ void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
}
}
+void HardwareUnitInfo::finalizeCycles() {
+ if (BufferSize == 0 || AllSUs.empty())
+ return;
+
+ // We estimate the amount of cycles it takes to free up a slot in the buffer
+ // as the average cycles per SU.
+ BufferCycles = TotalCycles / AllSUs.size();
+ // A single-entry buffer does not reduce TotalCycles.
+ if (BufferSize == 1)
+ return;
+
+ // The TotalCycles is normalized against the BufferSize.
+ // This provides an estimate of the TotalCycles which is not always accurate
+ // -- particularly in cases where we have fewer instructions than the
+ // BufferSize. For example, if we have 2 instructions which each take 50
+ // cycles and a BufferSize of 16, then a TotalCycles of 51 cycles would be
+ // somewhat accurate. This normalization calculates TotalCycles as 6. However,
+ // if we have 64 of these instructions, our normalized estimate of 200 is more
+ // reasonable, given the more accurate measure is 264. Having a completely
+ // accurate measure is not very important, since this metric is mainly used to
+ // compare the relative demand per HardwareUnit across the region. The simpler
+ // estimate makes managing the metric incrementally during scheduling much
+ // simpler.
+ TotalCycles /= BufferSize;
+}
+
HardwareUnitInfo *
CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
for (HardwareUnitInfo &HWUICand : HWUInfo) {
@@ -184,6 +214,10 @@ CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
unsigned CandidateHeuristics::getHWUICyclesForInst(SUnit *SU) {
assert(SchedModel && SchedModel->hasInstrSchedModel());
+ MachineInstr *MI = SU->getInstr();
+ if (SII->isDS(*MI))
+ return SchedModel->computeInstrLatency(MI);
+
unsigned ReleaseAtCycle = 0;
const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
for (TargetSchedModel::ProcResIter PI = SchedModel->getWriteProcResBegin(SC),
@@ -221,6 +255,7 @@ void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
HWUInfo[(int)InstructionFlavor::WMMA].setProducesCoexecWindow(true);
HWUInfo[(int)InstructionFlavor::MultiCycleVALU].setProducesCoexecWindow(true);
HWUInfo[(int)InstructionFlavor::TRANS].setProducesCoexecWindow(true);
+ HWUInfo[(int)InstructionFlavor::DS].setBufferSize(DefaultBufferSizes::DS);
collectHWUIPressure();
}
@@ -234,6 +269,9 @@ void CandidateHeuristics::collectHWUIPressure() {
HWUInfo[(int)(Flavor)].insert(&SU, getHWUICyclesForInst(&SU));
}
+ for (auto &HWUI : HWUInfo)
+ HWUI.finalizeCycles();
+
LLVM_DEBUG(dumpRegionSummary());
}
@@ -681,7 +719,26 @@ bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
SchedCandidate &TryCand,
- SchedBoundary &Zone) const {
+ SchedBoundary &Zone) {
+ auto getBufferFullStalls = [this, &Zone](SUnit *SU) -> unsigned {
+ InstructionFlavor Flavor = classifyFlavor(
+ *SU->getInstr(), *static_cast<const SIInstrInfo *>(DAG->TII));
+ HardwareUnitInfo *HWUI = Heurs.getHWUIFromFlavor(Flavor);
+
+ // A BufferSize of 0 is unlimited, so it has no FIFO scheduling cost.
+ if (HWUI->getBufferSize() == 0)
+ return 0;
+
+ // getBufferAvailableCycle assumes top-down scheduling.
+ assert(Zone.isTop());
+ unsigned CurrCycle = Zone.getCurrCycle();
+ unsigned BufferReadyCycle = HWUI->getBufferAvailableCycle(CurrCycle);
+ if (BufferReadyCycle <= CurrCycle)
+ return 0;
+
+ return BufferReadyCycle - CurrCycle;
+ };
+
// Treat structural and latency stalls as a single scheduling cost for the
// current cycle.
struct StallCosts {
@@ -689,6 +746,7 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
unsigned Structural = 0;
unsigned Latency = 0;
unsigned Effective = 0;
+ unsigned Buffer = 0;
};
unsigned CurrCycle = Zone.getCurrCycle();
@@ -698,7 +756,9 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
Costs.Structural = getStructuralStallCycles(Zone, SU);
Costs.Latency = Zone.getLatencyStallCycles(SU);
- Costs.Effective = std::max({Costs.Ready, Costs.Structural, Costs.Latency});
+ Costs.Buffer = getBufferFullStalls(SU);
+ Costs.Effective =
+ std::max({Costs.Ready, Costs.Structural, Costs.Latency, Costs.Buffer});
return Costs;
};
@@ -708,10 +768,11 @@ bool AMDGPUCoExecSchedStrategy::tryEffectiveStall(SchedCandidate &Cand,
LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
dbgs() << "Effective stalls: try=" << TryCosts.Effective
<< " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
- << ", lat=" << TryCosts.Latency << ") cand=" << CandCosts.Effective
- << " (ready=" << CandCosts.Ready
+ << ", lat=" << TryCosts.Latency << ", buffer=" << TryCosts.Buffer
+ << ") cand=" << CandCosts.Effective << " (ready=" << CandCosts.Ready
<< ", struct=" << CandCosts.Structural
- << ", lat=" << CandCosts.Latency << ")\n";
+ << ", lat=" << CandCosts.Latency << ", buffer=" << CandCosts.Buffer
+ << ")\n";
});
return tryLess(TryCosts.Effective, CandCosts.Effective, TryCand, Cand, Stall);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 4ea6853e46a64..672c5e3c33b97 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -21,6 +21,9 @@
namespace llvm {
namespace AMDGPU {
+namespace DefaultBufferSizes {
+constexpr unsigned DS = 16;
+} // namespace DefaultBufferSizes
/// AMDGPU-specific scheduling decision reasons. These provide more granularity
/// than the generic CandReason enum for debugging purposes.
@@ -66,6 +69,8 @@ class HardwareUnitInfo {
SmallSetVector<SUnit *, 16> PrioritySUs;
/// All the SUs in the region that consume this resource.
SmallSetVector<SUnit *, 16> AllSUs;
+ /// All the SUs for this HardwareUnit that have already been scheduled.
+ SmallVector<SUnit *, 16> ScheduledSUs;
/// The total number of busy cycles for this HardwareUnit for a given region.
unsigned TotalCycles = 0;
/// InstructionFlavor mapping.
@@ -75,6 +80,26 @@ class HardwareUnitInfo {
/// / MFMA instructions may take multiple cycles, which may be overlapped with
/// instructions on other HardwareUnits.
bool ProducesCoexecWindow = false;
+ /// How many instructions can be held simultaneously for this HardwareUnit.
+ /// A value of 0 means there is no limit. A value of 1 models an unbuffered
+ /// resource with a single in-flight instruction.
+ ///
+ /// This may approximate the hardware. For example, for LDS instructions
+ /// it is a well-known phenomena that oversubscribing the LDS unit results in
+ /// longer latency for the LDS instructions. While it is true that there is a
+ /// hard limit to the amount of simulatenous in-flight LDS instructions, good
+ /// scheduling would also cool off the LDS to avoid other forms of hardware
+ /// contention and increasing LDS latency. Thus, we limit the amount of LDS
+ /// instructions we are willing to schedule close together, though this does
+ /// not correspond 1:1 with a hardware mechanism.
+ unsigned BufferSize = 0;
+ /// How many cycles it takes for an instruction to clear the buffer.
+ ///
+ /// Again, this may be an apprxoimation. For example, for memory FIFOs, the
+ /// actual amount of cycles it will take to clear it is dependent on how
+ /// quickly prior instructions evacuate the FIFO, which is based on runtime
+ /// behavior which is not modelled in the compiler.
+ unsigned BufferCycles = 0;
public:
HardwareUnitInfo() {}
@@ -96,6 +121,24 @@ class HardwareUnitInfo {
bool contains(SUnit *SU) const { return AllSUs.contains(SU); }
+ void setBufferSize(unsigned Size) { BufferSize = Size; }
+
+ unsigned getBufferSize() { return BufferSize; }
+
+ /// \returns the next cycle where there is space in the buffer.
+ unsigned getBufferAvailableCycle(unsigned CurrCycle) {
+ // An unlimited buffer is always available.
+ if (BufferSize == 0)
+ return CurrCycle;
+
+ // Buffer is available now.
+ if (ScheduledSUs.size() < BufferSize)
+ return CurrCycle;
+
+ return BufferCycles +
+ ScheduledSUs[ScheduledSUs.size() - BufferSize]->TopReadyCycle;
+ }
+
/// \returns the SUnit with higher priority or nullptr if they are the same.
/// This method looks through the PrioritySUs to determine if one SU is more
/// prioritized than the other. If neither are in the PrioritySUs list, then
@@ -114,9 +157,12 @@ class HardwareUnitInfo {
void reset() {
AllSUs.clear();
PrioritySUs.clear();
+ ScheduledSUs.clear();
TotalCycles = 0;
Type = AMDGPU::InstructionFlavor::Other;
ProducesCoexecWindow = false;
+ BufferSize = 0;
+ BufferCycles = 0;
}
/// \returns the next SU in PrioritySUs that is not ready. If \p LookDeep is
@@ -136,6 +182,11 @@ class HardwareUnitInfo {
/// and reducing its \p BlockingCycles from the TotalCycles. This maintains
/// the list of PrioritySUs.
void markScheduled(SUnit *SU, unsigned BlockingCycles);
+ /// After we've collected all the region pressure for this HWUI, correct for
+ /// any specifics of the behavior of this resource. For example, if the
+ /// HardwareUnit can hold N instructions simultaneously, then there is no
+ /// penalty for scheduling N instructions back to back.
+ void finalizeCycles();
};
//===----------------------------------------------------------------------===//
@@ -160,10 +211,6 @@ class CandidateHeuristics {
/// SU.
unsigned getHWUICyclesForInst(SUnit *SU);
- /// Given a \p Flavor , find the corresponding HardwareUnit. \returns the
- /// mapped HardwareUnit.
- HardwareUnitInfo *getHWUIFromFlavor(AMDGPU::InstructionFlavor Flavor);
-
public:
CandidateHeuristics() = default;
@@ -173,7 +220,11 @@ class CandidateHeuristics {
/// Update the state to reflect that \p SU is going to be scheduled.
void updateForScheduling(SUnit *SU);
- /// Sort the HWUInfo vector. After sorting, the HardwareUnits that are highest
+ /// Given a \p Flavor , find the corresponding HardwareUnit. \returns the
+ /// mapped HardwareUnit.
+ HardwareUnitInfo *getHWUIFromFlavor(AMDGPU::InstructionFlavor Flavor);
+
+ /// Sort the HardwarUnitInfo vector. After sorting, the HWUI that are highest
/// priority are first. Priority is determined by maximizing coexecution and
/// keeping the critical HardwareUnit busy.
void sortHWUIResources();
@@ -202,7 +253,7 @@ class CandidateHeuristics {
class AMDGPUCoExecSchedStrategy final : public GCNSchedStrategy {
protected:
bool tryEffectiveStall(SchedCandidate &Cand, SchedCandidate &TryCand,
- SchedBoundary &Zone) const;
+ SchedBoundary &Zone);
AMDGPU::AMDGPUSchedReason LastAMDGPUReason = AMDGPU::AMDGPUSchedReason::None;
CandidateHeuristics Heurs;
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir b/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
index b4e8a86de06b8..43a735248ea5a 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
@@ -35,7 +35,7 @@
# DBG: CoExec window complete:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
-# DBG-NEXT: Scheduled: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 E E I E E I - V -
---
name: wmma_ds_salu_only
tracksRegLiveness: true
@@ -58,12 +58,12 @@ body: |
; CHECK-NEXT: early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF6]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_1]], [[DEF6]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 64, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF6]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_1]], [[DEF6]], implicit-def dead $scc
; CHECK-NEXT: [[S_ADD_I32_3:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_2]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[DS_READ_B128_gfx9_4]], implicit [[S_ADD_I32_3]], implicit %17
bb.0:
@@ -340,8 +340,8 @@ body: |
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
- ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF10]], [[DEF11]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF10]], [[DEF11]], implicit-def dead $scc
; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF8]], implicit $mode, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF10]], implicit-def dead $scc
; CHECK-NEXT: [[V_EXP_F32_e32_1:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF9]], implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
new file mode 100644
index 0000000000000..05a08205ea767
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-ds-fifo.mir
@@ -0,0 +1,48 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler %s -filetype=null 2>&1 | FileCheck --check-prefix=DEBUG %s
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck --check-prefix=ORDER %s
+
+# DEBUG: Effective stalls: try=4 (ready=0, struct=0, lat=0, buffer=4) cand=0 (ready=0, struct=0, lat=0, buffer=0)
+
+# ORDER-LABEL: name: ds_fifo_stall
+# ORDER: body: |
+# ORDER-NEXT: bb.0:
+# ORDER-NEXT: %0:vgpr_32 = IMPLICIT_DEF
+# ORDER-COUNT-16: DS_READ_B32
+# ORDER-NEXT: %18:vgpr_32 = V_MOV_B32_e32 %0, implicit $exec
+# ORDER-NEXT: %17:vgpr_32 = DS_READ_B32
+
+--- |
+ @lds = internal addrspace(3) global [17 x i32] undef
+
+ define void @ds_fifo_stall() #0 { ret void }
+
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
+...
+
+---
+name: ds_fifo_stall
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 0)`, addrspace 3)
+ %2:vgpr_32 = DS_READ_B32 %0, 4, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 1)`, addrspace 3)
+ %3:vgpr_32 = DS_READ_B32 %0, 8, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 2)`, addrspace 3)
+ %4:vgpr_32 = DS_READ_B32 %0, 12, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 3)`, addrspace 3)
+ %5:vgpr_32 = DS_READ_B32 %0, 16, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 4)`, addrspace 3)
+ %6:vgpr_32 = DS_READ_B32 %0, 20, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 5)`, addrspace 3)
+ %7:vgpr_32 = DS_READ_B32 %0, 24, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 6)`, addrspace 3)
+ %8:vgpr_32 = DS_READ_B32 %0, 28, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 7)`, addrspace 3)
+ %9:vgpr_32 = DS_READ_B32 %0, 32, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 8)`, addrspace 3)
+ %10:vgpr_32 = DS_READ_B32 %0, 36, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 9)`, addrspace 3)
+ %11:vgpr_32 = DS_READ_B32 %0, 40, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 10)`, addrspace 3)
+ %12:vgpr_32 = DS_READ_B32 %0, 44, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 11)`, addrspace 3)
+ %13:vgpr_32 = DS_READ_B32 %0, 48, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 12)`, addrspace 3)
+ %14:vgpr_32 = DS_READ_B32 %0, 52, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 13)`, addrspace 3)
+ %15:vgpr_32 = DS_READ_B32 %0, 56, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 14)`, addrspace 3)
+ %16:vgpr_32 = DS_READ_B32 %0, 60, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 15)`, addrspace 3)
+ %17:vgpr_32 = DS_READ_B32 %0, 64, 0, implicit $m0, implicit $exec :: (load (s32) from `ptr addrspace(3) getelementptr ([17 x i32], ptr addrspace(3) @lds, i32 0, i32 16)`, addrspace 3)
+ %18:vgpr_32 = V_MOV_B32_e32 %0, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18
+...
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index c730fc4c8674b..048b9abdb4c27 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -315,21 +315,21 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v124 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v124 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v124 offset:384
+; COEXEC-NEXT: s_wait_dscnt 0xf
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v124 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
-; COEXEC-NEXT: s_wait_dscnt 0x13
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
+; COEXEC-NEXT: s_wait_dscnt 0x12
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
-; COEXEC-NEXT: s_wait_dscnt 0x16
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
; COEXEC-NEXT: s_wait_dscnt 0x14
More information about the llvm-commits
mailing list