[llvm] [AMDGPU] Add heuristic to set CarriedLatencies (PR #224057)
Jeffrey Byrnes via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 16 08:56:39 PDT 2026
https://github.com/jrbyrnes created https://github.com/llvm/llvm-project/pull/224057
After https://github.com/llvm/llvm-project/commit/8723e4a1bc0b6078e35c17532952f79fb8480f53 users have some control to defer scheduling instructions which depend on loads from predecessor blocks. This PR adds a simple heuristic to control setting the Fence mode, which has shown to be the most useful in practice.
We would like to defer scheduling the ATOMIC_FENCE, if 1. there is a corresponding load/store in flight, and 2. there have not been enough cycles for that instruction to reasonably finish. Unfortunately, since we are reasoning about loads/stores in predecessor blocks, we can not reliably know where these will in the final schedule. However, we can inspect the dependency structure to identify cases where these instructions may be pushed back.
This heuristic identifies predecessor blocks in which ds_loads must be scheduled after all wmma instructions, as this dependency structure commonly leads to ds_loads at the end of the block. In such scenarios, we defer scheduling the fence to attempt to hide latency from incoming ds_loads.
Users may still override this heuristic as needed, but this heuristic has shown accuracy for mi450 ML kernels.
>From 2187ab225fb80b58c173273cb6c8def740ad3939 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes at amd.com>
Date: Mon, 14 Sep 2026 10:35:58 -0700
Subject: [PATCH] [AMDGPU] Add heuristic to set CarriedLatencies
Change-Id: I0a691cbf89de0320c8719a4b3fbe678e8fe72ba6
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 57 ++-
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 4 +
.../AMDGPU/coexec-sched-carried-latency.mir | 367 ++++++++++++++++++
3 files changed, 423 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/coexec-sched-carried-latency.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 6a098f36c43fa..6eabec6b84994 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -20,9 +20,6 @@ using namespace llvm;
using namespace llvm::AMDGPU;
#define DEBUG_TYPE "machine-scheduler"
-namespace {
-enum class CarriedLatency { Off, Fence, All };
-} // namespace
static cl::opt<CarriedLatency> BlockCarriedLatency(
"amdgpu-block-carried-latency", cl::Hidden, cl::init(CarriedLatency::Off),
@@ -627,7 +624,7 @@ void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
}
unsigned CandidateHeuristics::getCarriedLatency(SUnit *SU) {
- if (BlockCarriedLatency == CarriedLatency::Off)
+ if (RegionCarriedLatency == CarriedLatency::Off)
return 0;
MachineInstr *MI = SU->getInstr();
@@ -652,7 +649,7 @@ unsigned CandidateHeuristics::getCarriedLatency(SUnit *SU) {
}
}
- if (BlockCarriedLatency == CarriedLatency::Fence)
+ if (RegionCarriedLatency == CarriedLatency::Fence)
return CarriedLatency;
for (MachineOperand &Op : MI->all_uses()) {
@@ -710,6 +707,56 @@ void CandidateHeuristics::collectRegionSummary() {
if (!SchedModel || !SchedModel->hasInstrSchedModel())
return;
+ if (BlockCarriedLatency.getNumOccurrences())
+ RegionCarriedLatency = BlockCarriedLatency;
+
+ if (!BlockCarriedLatency.getNumOccurrences()) {
+ SmallVector<SUnit *, 16> RegionWMMAs;
+
+ for (auto &SU : DAG->SUnits) {
+ if (!SU.getInstr())
+ continue;
+ MachineInstr *MI = SU.getInstr();
+ const InstructionFlavor Flavor = classifyFlavor(*MI, *SII);
+ if (Flavor == InstructionFlavor::WMMA)
+ RegionWMMAs.push_back(&SU);
+ }
+
+ bool MustHaveDSAfter = RegionWMMAs.size();
+
+ // In some cases, we may end up with loads at the end of a predecssor block.
+ // In these cases, it is preferable to defer scheduling a fence which
+ // corresponds with a waitcnt for those loads until the latency of the load
+ // has cleared. Unfortunately, there is no reliable way to determine whether
+ // or not a predecessor block will end up scheduling loads at the end. Here,
+ // we inspect the dependency structure to define a rough heuristic: if we
+ // must schedule ds_loads after wmma, then we carry the latency of ds_loads
+ // to successor block fences.
+ // TODO: 1. extend to different memory instructions, 2. teach carried
+ // latencies about fence legalization.
+ for (SUnit *SU : RegionWMMAs) {
+ bool HasDSSucc = false;
+ for (auto &Succ : SU->Succs) {
+ if (!Succ.getSUnit()->getInstr())
+ continue;
+ if (classifyFlavor(*Succ.getSUnit()->getInstr(), *SII) ==
+ InstructionFlavor::DS) {
+ HasDSSucc = true;
+ break;
+ }
+ }
+
+ if (!HasDSSucc) {
+ MustHaveDSAfter = false;
+ break;
+ }
+ }
+
+ if (MustHaveDSAfter) {
+ RegionCarriedLatency = CarriedLatency::Fence;
+ }
+ }
+
for (auto &SU : DAG->SUnits) {
MachineInstr *MI = SU.getInstr();
const InstructionFlavor Flavor = classifyFlavor(*MI, *SII);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 9032df3724ab1..9a59e871b2354 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -25,6 +25,8 @@ namespace DefaultBufferSizes {
constexpr unsigned DS = 16;
} // namespace DefaultBufferSizes
+enum class CarriedLatency { Off, Fence, All };
+
/// AMDGPU-specific scheduling decision reasons. These provide more granularity
/// than the generic CandReason enum for debugging purposes.
enum class AMDGPUSchedReason : uint8_t {
@@ -203,6 +205,8 @@ class CandidateHeuristics {
const SIRegisterInfo *SRI;
const TargetSchedModel *SchedModel;
SmallVector<HardwareUnitInfo, 8> HWUInfo;
+
+ AMDGPU::CarriedLatency RegionCarriedLatency = AMDGPU::CarriedLatency::Off;
DenseMap<MachineInstr *, unsigned> CarriedLatencies;
/// Walk over the region and collect characteristics for the various
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-carried-latency.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-carried-latency.mir
new file mode 100644
index 0000000000000..47bab854caab1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-carried-latency.mir
@@ -0,0 +1,367 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -amdgpu-block-carried-latency=fence %s -o - | FileCheck --check-prefix=DEFAULT %s
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck --check-prefix=FENCE %s
+
+---
+name: same_iteration_loop
+tracksRegLiveness: true
+body: |
+ ; DEFAULT-LABEL: name: same_iteration_loop
+ ; DEFAULT: bb.0:
+ ; DEFAULT-NEXT: successors: %bb.1(0x80000000)
+ ; DEFAULT-NEXT: liveins: $vgpr0, $sgpr0, $sgpr1
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; DEFAULT-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DEFAULT-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: S_BRANCH %bb.1
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: bb.1:
+ ; DEFAULT-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def $scc
+ ; DEFAULT-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = COPY [[S_ADD_I32_]]
+ ; DEFAULT-NEXT: S_CMP_LT_U32 [[S_ADD_I32_]], [[COPY1]], implicit-def $scc
+ ; DEFAULT-NEXT: ATOMIC_FENCE 5, 2
+ ; DEFAULT-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; DEFAULT-NEXT: S_BARRIER_WAIT -1
+ ; DEFAULT-NEXT: ATOMIC_FENCE 4, 2
+ ; DEFAULT-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; DEFAULT-NEXT: ATOMIC_FENCE 5, 2
+ ; DEFAULT-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; DEFAULT-NEXT: S_BARRIER_WAIT -1
+ ; DEFAULT-NEXT: ATOMIC_FENCE 4, 2
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; DEFAULT-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_]], 8, [[DS_READ_B128_gfx9_2]], 8, [[DEF2]], 0, 0, implicit $exec
+ ; DEFAULT-NEXT: early-clobber %11:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_1]], 8, [[DS_READ_B128_gfx9_3]], 8, %10, 0, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = COPY %11
+ ; DEFAULT-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: bb.2:
+ ; DEFAULT-NEXT: S_ENDPGM 0
+ ;
+ ; FENCE-LABEL: name: same_iteration_loop
+ ; FENCE: bb.0:
+ ; FENCE-NEXT: successors: %bb.1(0x80000000)
+ ; FENCE-NEXT: liveins: $vgpr0, $sgpr0, $sgpr1
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; FENCE-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; FENCE-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; FENCE-NEXT: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+ ; FENCE-NEXT: [[DEF1:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+ ; FENCE-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; FENCE-NEXT: S_BRANCH %bb.1
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: bb.1:
+ ; FENCE-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: ATOMIC_FENCE 5, 2
+ ; FENCE-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; FENCE-NEXT: S_BARRIER_WAIT -1
+ ; FENCE-NEXT: ATOMIC_FENCE 4, 2
+ ; FENCE-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; FENCE-NEXT: ATOMIC_FENCE 5, 2
+ ; FENCE-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; FENCE-NEXT: [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def $scc
+ ; FENCE-NEXT: S_BARRIER_WAIT -1
+ ; FENCE-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = COPY [[S_ADD_I32_]]
+ ; FENCE-NEXT: S_CMP_LT_U32 [[S_ADD_I32_]], [[COPY1]], implicit-def $scc
+ ; FENCE-NEXT: ATOMIC_FENCE 4, 2
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; FENCE-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_]], 8, [[DS_READ_B128_gfx9_2]], 8, [[DEF2]], 0, 0, implicit $exec
+ ; FENCE-NEXT: early-clobber %11:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_1]], 8, [[DS_READ_B128_gfx9_3]], 8, %10, 0, 0, implicit $exec
+ ; FENCE-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = COPY %11
+ ; FENCE-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: bb.2:
+ ; FENCE-NEXT: S_ENDPGM 0
+ ; CHECK-LABEL: name: same_iteration_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr0, $sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def $scc
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = COPY [[S_ADD_I32_]]
+ ; CHECK-NEXT: S_CMP_LT_U32 [[S_ADD_I32_]], [[COPY1]], implicit-def $scc
+ ; CHECK-NEXT: ATOMIC_FENCE 5, 2
+ ; CHECK-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; CHECK-NEXT: S_BARRIER_WAIT -1
+ ; CHECK-NEXT: ATOMIC_FENCE 4, 2
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ATOMIC_FENCE 5, 2
+ ; CHECK-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; CHECK-NEXT: S_BARRIER_WAIT -1
+ ; CHECK-NEXT: ATOMIC_FENCE 4, 2
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_]], 8, [[DS_READ_B128_gfx9_2]], 8, [[DEF2]], 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber %11:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_1]], 8, [[DS_READ_B128_gfx9_3]], 8, %10, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = COPY %11
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0, $sgpr0, $sgpr1
+
+ %0:sgpr_128 = IMPLICIT_DEF
+ %1:sgpr_256 = IMPLICIT_DEF
+ %2:vgpr_32 = COPY $vgpr0
+ %3:sgpr_32 = COPY $sgpr0
+ %4:sgpr_32 = S_MOV_B32 0
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ ATOMIC_FENCE 5, 2
+ S_BARRIER_SIGNAL_IMM -1
+ S_BARRIER_WAIT -1
+ ATOMIC_FENCE 4, 2
+ TENSOR_LOAD_TO_LDS_d2_gfx1250 %0, %1, 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+
+ ATOMIC_FENCE 5, 2
+ S_BARRIER_SIGNAL_IMM -1
+ S_BARRIER_WAIT -1
+ ATOMIC_FENCE 4, 2
+ %10:vreg_128_align2 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %2, 16, 0, implicit $exec
+ %12:vreg_128_align2 = DS_READ_B128_gfx9 %2, 32, 0, implicit $exec
+ %13:vreg_128_align2 = DS_READ_B128_gfx9 %2, 48, 0, implicit $exec
+ %14:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, %10, 8, %12, 8, %5, 0, 0, implicit $exec
+ %15:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, %11, 8, %13, 8, %14, 0, 0, implicit $exec
+ %20:sgpr_32 = S_ADD_I32 %4, 1, implicit-def $scc
+ S_CMP_LT_U32 %20, %3, implicit-def $scc
+ %4:sgpr_32 = COPY %20
+ %5:vreg_256_align2 = COPY %15
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+ bb.2:
+ S_ENDPGM 0
+...
+
+---
+name: prefetch_loop
+tracksRegLiveness: true
+body: |
+ ; DEFAULT-LABEL: name: prefetch_loop
+ ; DEFAULT: bb.0:
+ ; DEFAULT-NEXT: successors: %bb.1(0x80000000)
+ ; DEFAULT-NEXT: liveins: $vgpr0, $sgpr0, $sgpr1
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; DEFAULT-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DEFAULT-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; DEFAULT-NEXT: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: [[DEF1:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; DEFAULT-NEXT: ATOMIC_FENCE 5, 2
+ ; DEFAULT-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; DEFAULT-NEXT: S_BARRIER_WAIT -1
+ ; DEFAULT-NEXT: ATOMIC_FENCE 4, 2
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; DEFAULT-NEXT: S_BRANCH %bb.1
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: bb.1:
+ ; DEFAULT-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_]], 8, [[DS_READ_B128_gfx9_2]], 8, [[DEF2]], 0, 0, implicit $exec
+ ; DEFAULT-NEXT: [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def $scc
+ ; DEFAULT-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = COPY [[S_ADD_I32_]]
+ ; DEFAULT-NEXT: S_CMP_LT_U32 [[S_ADD_I32_]], [[COPY1]], implicit-def $scc
+ ; DEFAULT-NEXT: early-clobber %11:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_1]], 8, [[DS_READ_B128_gfx9_3]], 8, %10, 0, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = COPY %11
+ ; DEFAULT-NEXT: ATOMIC_FENCE 5, 2
+ ; DEFAULT-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; DEFAULT-NEXT: S_BARRIER_WAIT -1
+ ; DEFAULT-NEXT: ATOMIC_FENCE 4, 2
+ ; DEFAULT-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; DEFAULT-NEXT: ATOMIC_FENCE 5, 2
+ ; DEFAULT-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; DEFAULT-NEXT: S_BARRIER_WAIT -1
+ ; DEFAULT-NEXT: ATOMIC_FENCE 4, 2
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; DEFAULT-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; DEFAULT-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; DEFAULT-NEXT: {{ $}}
+ ; DEFAULT-NEXT: bb.2:
+ ; DEFAULT-NEXT: S_ENDPGM 0
+ ;
+ ; FENCE-LABEL: name: prefetch_loop
+ ; FENCE: bb.0:
+ ; FENCE-NEXT: successors: %bb.1(0x80000000)
+ ; FENCE-NEXT: liveins: $vgpr0, $sgpr0, $sgpr1
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; FENCE-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; FENCE-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; FENCE-NEXT: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+ ; FENCE-NEXT: [[DEF1:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+ ; FENCE-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; FENCE-NEXT: ATOMIC_FENCE 5, 2
+ ; FENCE-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; FENCE-NEXT: S_BARRIER_WAIT -1
+ ; FENCE-NEXT: ATOMIC_FENCE 4, 2
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; FENCE-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; FENCE-NEXT: S_BRANCH %bb.1
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: bb.1:
+ ; FENCE-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_]], 8, [[DS_READ_B128_gfx9_2]], 8, [[DEF2]], 0, 0, implicit $exec
+ ; FENCE-NEXT: [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def $scc
+ ; FENCE-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = COPY [[S_ADD_I32_]]
+ ; FENCE-NEXT: S_CMP_LT_U32 [[S_ADD_I32_]], [[COPY1]], implicit-def $scc
+ ; FENCE-NEXT: early-clobber %11:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_1]], 8, [[DS_READ_B128_gfx9_3]], 8, %10, 0, 0, implicit $exec
+ ; FENCE-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = COPY %11
+ ; FENCE-NEXT: ATOMIC_FENCE 5, 2
+ ; FENCE-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; FENCE-NEXT: S_BARRIER_WAIT -1
+ ; FENCE-NEXT: ATOMIC_FENCE 4, 2
+ ; FENCE-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; FENCE-NEXT: ATOMIC_FENCE 5, 2
+ ; FENCE-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; FENCE-NEXT: S_BARRIER_WAIT -1
+ ; FENCE-NEXT: ATOMIC_FENCE 4, 2
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; FENCE-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; FENCE-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; FENCE-NEXT: {{ $}}
+ ; FENCE-NEXT: bb.2:
+ ; FENCE-NEXT: S_ENDPGM 0
+ ; CHECK-LABEL: name: prefetch_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr0, $sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ATOMIC_FENCE 5, 2
+ ; CHECK-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; CHECK-NEXT: S_BARRIER_WAIT -1
+ ; CHECK-NEXT: ATOMIC_FENCE 4, 2
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_]], 8, [[DS_READ_B128_gfx9_2]], 8, [[DEF2]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def $scc
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = COPY [[S_ADD_I32_]]
+ ; CHECK-NEXT: S_CMP_LT_U32 [[S_ADD_I32_]], [[COPY1]], implicit-def $scc
+ ; CHECK-NEXT: early-clobber %11:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, [[DS_READ_B128_gfx9_1]], 8, [[DS_READ_B128_gfx9_3]], 8, %10, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = COPY %11
+ ; CHECK-NEXT: ATOMIC_FENCE 5, 2
+ ; CHECK-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; CHECK-NEXT: S_BARRIER_WAIT -1
+ ; CHECK-NEXT: ATOMIC_FENCE 4, 2
+ ; CHECK-NEXT: TENSOR_LOAD_TO_LDS_d2_gfx1250 [[DEF]], [[DEF1]], 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ; CHECK-NEXT: ATOMIC_FENCE 5, 2
+ ; CHECK-NEXT: S_BARRIER_SIGNAL_IMM -1
+ ; CHECK-NEXT: S_BARRIER_WAIT -1
+ ; CHECK-NEXT: ATOMIC_FENCE 4, 2
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_align2 = DS_READ_B128_gfx9 [[COPY]], 48, 0, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0, $sgpr0, $sgpr1
+
+ %0:sgpr_128 = IMPLICIT_DEF
+ %1:sgpr_256 = IMPLICIT_DEF
+ %2:vgpr_32 = COPY $vgpr0
+ %3:sgpr_32 = COPY $sgpr0
+ %4:sgpr_32 = S_MOV_B32 0
+ %5:vreg_256_align2 = IMPLICIT_DEF
+
+ TENSOR_LOAD_TO_LDS_d2_gfx1250 %0, %1, 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+ ATOMIC_FENCE 5, 2
+ S_BARRIER_SIGNAL_IMM -1
+ S_BARRIER_WAIT -1
+ ATOMIC_FENCE 4, 2
+ %10:vreg_128_align2 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %2, 16, 0, implicit $exec
+ %12:vreg_128_align2 = DS_READ_B128_gfx9 %2, 32, 0, implicit $exec
+ %13:vreg_128_align2 = DS_READ_B128_gfx9 %2, 48, 0, implicit $exec
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %20:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, %10, 8, %12, 8, %5, 0, 0, implicit $exec
+ %21:vreg_256_align2 = V_WMMA_F32_16X16X16_F16_w32_twoaddr 8, %11, 8, %13, 8, %20, 0, 0, implicit $exec
+ ATOMIC_FENCE 5, 2
+ S_BARRIER_SIGNAL_IMM -1
+ S_BARRIER_WAIT -1
+ ATOMIC_FENCE 4, 2
+ TENSOR_LOAD_TO_LDS_d2_gfx1250 %0, %1, 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
+
+ ATOMIC_FENCE 5, 2
+ S_BARRIER_SIGNAL_IMM -1
+ S_BARRIER_WAIT -1
+ ATOMIC_FENCE 4, 2
+ %10:vreg_128_align2 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %2, 16, 0, implicit $exec
+ %12:vreg_128_align2 = DS_READ_B128_gfx9 %2, 32, 0, implicit $exec
+ %13:vreg_128_align2 = DS_READ_B128_gfx9 %2, 48, 0, implicit $exec
+ %40:sgpr_32 = S_ADD_I32 %4, 1, implicit-def $scc
+ S_CMP_LT_U32 %40, %3, implicit-def $scc
+ %4:sgpr_32 = COPY %40
+ %5:vreg_256_align2 = COPY %21
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+ bb.2:
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list