[llvm] [AMDGPU] Model WMMA co-execution windows in the scheduler for gfx1250 (PR #204077)
Austin Kerbow via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 14:50:32 PDT 2026
https://github.com/kerbowa updated https://github.com/llvm/llvm-project/pull/204077
>From db3b067a85ba5978749fe9b1f9b941eeec43ad3c Mon Sep 17 00:00:00 2001
From: Austin Kerbow <Austin.Kerbow at amd.com>
Date: Tue, 16 Jun 2026 00:04:25 -0500
Subject: [PATCH] [AMDGPU] Model WMMA co-execution windows in the scheduler for
gfx1250
WMMA instructions in gfx1250 expose an execution window during which
only certain other instruction classes may co-execute. Teach the hazard
recognizer about those windows so the scheduler can fill co-execution slots and
account for the resulting stalls. This adds a preRA hazard recognizer
mode.
Add AMDGPUCoExecInfo.h, a shared model of a co-execution window: the
per-stage capability bitmask, the stage types (CoExecStageType), and
CoExecInfo, which maps a multi-cycle instruction to its per-cycle slot
pattern via getCoExecInfo(). InstructionFlavor and its helpers move here
from AMDGPUCoExecSchedStrategy.h with no functional change so they can
be shared by the scheduler and the hazard recognizer.
---
llvm/include/llvm/CodeGen/MachineScheduler.h | 4 +-
llvm/lib/CodeGen/MachineScheduler.cpp | 28 +-
llvm/lib/Target/AMDGPU/AMDGPUCoExecInfo.h | 463 ++++++++++++++++
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 12 +
.../Target/AMDGPU/AMDGPUCoExecSchedStrategy.h | 99 +---
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 413 +++++++++++++-
llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h | 133 ++++-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 9 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 21 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +
.../CodeGen/AMDGPU/coexec-hazardrec-preRA.mir | 505 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 3 +-
llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll | 12 +-
.../llvm.amdgcn.sched.group.barrier.gfx12.ll | 5 +-
.../misched-into-wmma-hazard-shadow.mir | 4 +-
.../AMDGPU/wmma-trans-multi-shadow-hazard.mir | 67 +++
16 files changed, 1634 insertions(+), 148 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUCoExecInfo.h
create mode 100644 llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/wmma-trans-multi-shadow-hazard.mir
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index a6b92934a78eb..5280a2376c34a 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -88,6 +88,7 @@
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/ScheduleDAGInstrs.h"
#include "llvm/CodeGen/ScheduleDAGMutation.h"
+#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
@@ -134,7 +135,6 @@ class MachineInstr;
class MachineLoopInfo;
class RegisterClassInfo;
class SchedDFSResult;
-class ScheduleHazardRecognizer;
class TargetInstrInfo;
class TargetPassConfig;
class TargetRegisterInfo;
@@ -874,7 +874,7 @@ class SchedBoundary {
ReadyQueue Available;
ReadyQueue Pending;
- ScheduleHazardRecognizer *HazardRec = nullptr;
+ std::unique_ptr<ScheduleHazardRecognizer> HazardRec;
private:
/// True if the pending Q should be checked/updated before scheduling another
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index a92b1086a8438..e729c0f5b8af2 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -2466,7 +2466,7 @@ void CopyConstrain::apply(ScheduleDAGInstrs *DAGInstrs) {
static const unsigned InvalidCycle = ~0U;
-SchedBoundary::~SchedBoundary() { delete HazardRec; }
+SchedBoundary::~SchedBoundary() = default;
/// Given a Count of resource usage and a Latency value, return true if a
/// SchedBoundary becomes resource limited.
@@ -2485,10 +2485,8 @@ void SchedBoundary::reset() {
// A new HazardRec is created for each DAG and owned by SchedBoundary.
// Destroying and reconstructing it is very expensive though. So keep
// invalid, placeholder HazardRecs.
- if (HazardRec && HazardRec->isEnabled()) {
- delete HazardRec;
- HazardRec = nullptr;
- }
+ if (HazardRec && HazardRec->isEnabled())
+ HazardRec.reset();
Available.clear();
Pending.clear();
CheckPending = false;
@@ -3649,12 +3647,10 @@ void GenericScheduler::initialize(ScheduleDAGMI *dag) {
// Initialize the HazardRecognizers. If itineraries don't exist, are empty, or
// are disabled, then these HazardRecs will be disabled.
const InstrItineraryData *Itin = SchedModel->getInstrItineraries();
- if (!Top.HazardRec) {
- Top.HazardRec = DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG);
- }
- if (!Bot.HazardRec) {
- Bot.HazardRec = DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG);
- }
+ if (!Top.HazardRec)
+ Top.HazardRec.reset(DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG));
+ if (!Bot.HazardRec)
+ Bot.HazardRec.reset(DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG));
TopCand.SU = nullptr;
BotCand.SU = nullptr;
@@ -4314,12 +4310,10 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) {
// Initialize the HazardRecognizers. If itineraries don't exist, are empty,
// or are disabled, then these HazardRecs will be disabled.
const InstrItineraryData *Itin = SchedModel->getInstrItineraries();
- if (!Top.HazardRec) {
- Top.HazardRec = DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG);
- }
- if (!Bot.HazardRec) {
- Bot.HazardRec = DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG);
- }
+ if (!Top.HazardRec)
+ Top.HazardRec.reset(DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG));
+ if (!Bot.HazardRec)
+ Bot.HazardRec.reset(DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG));
TopClusterID = InvalidClusterId;
BotClusterID = InvalidClusterId;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecInfo.h
new file mode 100644
index 0000000000000..6f0f50ed7472b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecInfo.h
@@ -0,0 +1,463 @@
+//===-- AMDGPUCoExecInfo.h - Co-execution info ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Shared types for co-execution modeling used by GCNHazardRecognizer and the
+/// schedulers.
+///
+/// Multi-cycle instructions (WMMA, TRANS, etc.) have execution windows where
+/// other instruction types can co-execute. For WMMA, slot patterns depend on
+/// the variant:
+///
+/// E0 (Issue): Control instructions only (s_delay_alu, s_set_vgpr_msb)
+/// E (External): Memory and SALU can co-execute, no VALU
+/// I (Internal): VALU, TRANS, memory, and SALU can all co-execute
+/// V (Vacant): Memory/SALU/next-WMMA ok, NO VALU/TRANS
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
+
+#include "SIDefines.h"
+#include "SIInstrInfo.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/StringRef.h"
+#include <cassert>
+#include <cstdint>
+#include <optional>
+
+namespace llvm {
+
+namespace AMDGPU {
+
+//===----------------------------------------------------------------------===//
+// Co-execution Bitmasks
+//===----------------------------------------------------------------------===//
+
+/// Bitmask for instruction types allowed to co-execute at a stage.
+enum class CoExecMask : uint16_t {
+ None = 0,
+ CTRL = 1 << 0, // Control: s_delay_alu, s_set_vgpr_msb
+ VALU = 1 << 1, // Vector ALU
+ TRANS = 1 << 2, // Transcendentals (V_EXP etc)
+ SALU = 1 << 3, // Scalar ALU
+ DS = 1 << 4, // LDS read/write
+ VMEM = 1 << 5, // Global memory
+ SMEM = 1 << 6, // Scalar memory
+ WMMA = 1 << 7, // Next WMMA (V stages only)
+ All = 0xFFFF,
+
+ MEM = DS | VMEM | SMEM,
+ StageE0 = CTRL, // Issue: control only
+ StageE = CTRL | SALU | MEM, // External: mem/salu
+ StageI = CTRL | SALU | MEM | VALU | TRANS, // Internal: all ALU
+ // Internal + scaled-WMMA absorb: same as StageI but the next scaled
+ // WMMA may issue here - its LD_SCALE consumes the I cycle and the matrix
+ // multiply lands in the V slot that follows. Used for the last I before
+ // V of scaled patterns.
+ StageIS = StageI | WMMA,
+ StageV = CTRL | SALU | MEM | WMMA, // Vacant: no valu/trans
+ StageTR = All & ~TRANS, // TRANS co-exec: no TRANS
+
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All)
+};
+
+using CoExecMaskT = CoExecMask;
+
+//===----------------------------------------------------------------------===//
+// Instruction Flavor Classification
+//===----------------------------------------------------------------------===//
+
+/// Classification of instructions by execution characteristics.
+/// Used for scheduling decisions and co-execution slot preferences.
+enum class InstructionFlavor : uint8_t {
+ WMMA, // WMMA/MFMA matrix operations
+ SingleCycleVALU, // Single-cycle VALU (not TRANS, not multi-cycle CVT)
+ TRANS, // Transcendental ops (v_exp, v_log, etc.)
+ MultiCycleVALU, // VALU instructions with repeat rate > 1
+ VMEM, // FLAT/GLOBAL memory operations
+ SMEM, // Scalar memory operations
+ DS, // LDS/GDS operations
+ SALU, // Scalar ALU
+ DMA, // Tensor DMA operations
+ Fence, // Fences and waits
+ Other, // Everything else
+ NUM_FLAVORS
+};
+
+constexpr StringRef getFlavorName(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return "WMMA";
+ case InstructionFlavor::SingleCycleVALU:
+ return "VALU(1c)";
+ case InstructionFlavor::TRANS:
+ return "TRANS";
+ case InstructionFlavor::MultiCycleVALU:
+ return "VALU(Nc)";
+ case InstructionFlavor::VMEM:
+ return "VMEM";
+ case InstructionFlavor::SMEM:
+ return "SMEM";
+ case InstructionFlavor::DS:
+ return "DS";
+ case InstructionFlavor::SALU:
+ return "SALU";
+ case InstructionFlavor::DMA:
+ return "DMA";
+ case InstructionFlavor::Fence:
+ return "Fence";
+ case InstructionFlavor::Other:
+ return "Other";
+ case InstructionFlavor::NUM_FLAVORS:
+ return "???";
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+/// Classify \p MI into the execution flavor that drives both the scheduler's
+/// slot preferences and the hazard recognizer's co-execution masks.
+InstructionFlavor classifyFlavor(const MachineInstr &MI,
+ const SIInstrInfo &SII);
+
+/// Map a flavor to the co-execution class it occupies in a window slot.
+constexpr CoExecMaskT getCoExecMask(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return CoExecMask::WMMA;
+ case InstructionFlavor::TRANS:
+ return CoExecMask::TRANS;
+ case InstructionFlavor::SingleCycleVALU:
+ case InstructionFlavor::MultiCycleVALU:
+ // LDS DMA and tensor DMA issue on the VALU pipe.
+ case InstructionFlavor::DMA:
+ return CoExecMask::VALU;
+ case InstructionFlavor::DS:
+ return CoExecMask::DS;
+ case InstructionFlavor::VMEM:
+ return CoExecMask::VMEM;
+ case InstructionFlavor::SMEM:
+ return CoExecMask::SMEM;
+ case InstructionFlavor::SALU:
+ // Fences are s_barrier_*/s_wait_*, which issue on the scalar pipe.
+ case InstructionFlavor::Fence:
+ return CoExecMask::SALU;
+ case InstructionFlavor::Other:
+ return CoExecMask::CTRL;
+ case InstructionFlavor::NUM_FLAVORS:
+ break;
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+//===----------------------------------------------------------------------===//
+// Co-execution Stage Type
+//===----------------------------------------------------------------------===//
+
+/// Stage type for co-execution (for annotation/display).
+enum class CoExecStageType : uint8_t {
+ NONE = 0, // Not in co-exec window
+ E0, // Issue cycle - control only
+ E, // External - MEM/SALU allowed
+ I, // Internal - MEM/SALU/VALU allowed
+ IS, // Internal + scaled-WMMA absorb (I plus next-WMMA issue)
+ V, // Vacant - MEM/SALU/WMMA allowed, no VALU
+ TR // TRANS co-exec - everything except TRANS
+};
+
+inline const char *getStageTypeName(CoExecStageType T) {
+ switch (T) {
+ case CoExecStageType::NONE:
+ return "--";
+ case CoExecStageType::E0:
+ return "E0";
+ case CoExecStageType::E:
+ return "E";
+ case CoExecStageType::I:
+ return "I";
+ case CoExecStageType::IS:
+ return "IS";
+ case CoExecStageType::V:
+ return "V";
+ case CoExecStageType::TR:
+ return "TR";
+ }
+ llvm_unreachable("Unknown CoExecStageType");
+}
+
+/// Return a human-readable name for a mask holding a single instruction class,
+/// as produced by getCoExecMask().
+inline const char *getCoExecMaskName(CoExecMaskT Mask) {
+ switch (Mask) {
+ case CoExecMask::CTRL:
+ return "CTRL";
+ case CoExecMask::VALU:
+ return "VALU";
+ case CoExecMask::TRANS:
+ return "TRANS";
+ case CoExecMask::SALU:
+ return "SALU";
+ case CoExecMask::DS:
+ return "DS";
+ case CoExecMask::VMEM:
+ return "VMEM";
+ case CoExecMask::SMEM:
+ return "SMEM";
+ case CoExecMask::WMMA:
+ return "WMMA";
+ default:
+ llvm_unreachable("Not a single instruction class");
+ }
+}
+
+/// Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
+constexpr unsigned MaxCoExecStages = 32;
+
+//===----------------------------------------------------------------------===//
+// Co-execution Slot Info
+//===----------------------------------------------------------------------===//
+
+/// Per-slot info: which instruction classes may co-execute here.
+struct CoExecSlotInfo {
+ CoExecMaskT Mask = CoExecMask::All; // What CAN execute (correctness)
+};
+
+//===----------------------------------------------------------------------===//
+// Co-execution Info
+//===----------------------------------------------------------------------===//
+
+/// Co-execution characteristics for a multi-cycle instruction.
+struct CoExecInfo {
+ /// Number of cycles in the co-execution window, counting any trailing
+ /// vacant stages.
+ unsigned TotalWindow = 0;
+ /// Per-stage slot info (capability mask).
+ CoExecSlotInfo Slots[MaxCoExecStages];
+ /// Pattern string for display (e.g., "0EIIEEIIV").
+ StringRef Pattern;
+
+ /// Default constructor - initialize to safe defaults.
+ CoExecInfo() {
+ for (unsigned I = 0; I < MaxCoExecStages; ++I)
+ Slots[I].Mask = CoExecMask::All; // Default: permissive
+ }
+
+ /// Get capability mask for a stage.
+ CoExecMaskT getMask(unsigned Stage) const {
+ return Stage < TotalWindow ? Slots[Stage].Mask : CoExecMask::All;
+ }
+
+ /// Check if an instruction class mask can co-execute at a given stage.
+ bool canCoExec(CoExecMaskT InstMask, unsigned Stage) const {
+ if (Stage >= TotalWindow)
+ return true;
+ return any(Slots[Stage].Mask & InstMask);
+ }
+
+ /// Find next stage where the instruction class is allowed.
+ std::optional<unsigned> findNextAllowedStage(CoExecMaskT InstMask,
+ unsigned FromStage) const {
+ for (unsigned I = FromStage; I < TotalWindow; ++I) {
+ if (any(Slots[I].Mask & InstMask))
+ return I;
+ }
+ return std::nullopt;
+ }
+
+ /// Get stage type from mask for display.
+ static CoExecStageType getStageType(CoExecMaskT Mask) {
+ if (Mask == CoExecMask::StageE0)
+ return CoExecStageType::E0;
+ if (Mask == CoExecMask::StageE)
+ return CoExecStageType::E;
+ if (Mask == CoExecMask::StageIS)
+ return CoExecStageType::IS;
+ if (Mask == CoExecMask::StageI)
+ return CoExecStageType::I;
+ if (Mask == CoExecMask::StageV)
+ return CoExecStageType::V;
+ if (Mask == CoExecMask::StageTR)
+ return CoExecStageType::TR;
+ // For 'All' or unknown, return based on what's allowed.
+ if (any(Mask & CoExecMask::VALU))
+ return CoExecStageType::I; // If VALU allowed, it's I-like
+ if (any(Mask & CoExecMask::WMMA))
+ return CoExecStageType::V; // If WMMA allowed (not VALU), V-like
+ return CoExecStageType::E; // Default to E
+ }
+
+ /// Get stage type for a specific stage.
+ CoExecStageType getType(unsigned Stage) const {
+ return getStageType(getMask(Stage));
+ }
+
+ /// Build a CoExecInfo from a pattern string.
+ static CoExecInfo build(unsigned TotalWindow, const char *Pattern);
+};
+
+//===----------------------------------------------------------------------===//
+// Co-execution Info Construction
+//===----------------------------------------------------------------------===//
+
+/// Build CoExecInfo from a pattern string.
+/// Pattern chars: '0'=E0, 'E'=External, 'I'=Internal, 'V'=Vacant,
+/// 'S'=Internal+ScaleWMMAAbsorb (I plus next scaled WMMA),
+/// 'T'=TRANS co-exec (all except TRANS), 'A'=Any
+inline CoExecInfo CoExecInfo::build(unsigned TotalWindow, const char *Pattern) {
+ CoExecInfo Info;
+ Info.TotalWindow = TotalWindow;
+ Info.Pattern = Pattern;
+ assert(Info.Pattern.size() == TotalWindow &&
+ "Pattern must describe every cycle of the co-execution window");
+ assert(TotalWindow <= MaxCoExecStages && "Co-execution window is too long");
+
+ for (unsigned I = 0; I < Info.TotalWindow; ++I) {
+ switch (Pattern[I]) {
+ case '0':
+ Info.Slots[I].Mask = CoExecMask::StageE0;
+ break;
+ case 'E':
+ Info.Slots[I].Mask = CoExecMask::StageE;
+ break;
+ case 'I':
+ Info.Slots[I].Mask = CoExecMask::StageI;
+ break;
+ case 'S':
+ Info.Slots[I].Mask = CoExecMask::StageIS;
+ break;
+ case 'V':
+ Info.Slots[I].Mask = CoExecMask::StageV;
+ break;
+ case 'T':
+ Info.Slots[I].Mask = CoExecMask::StageTR;
+ break;
+ case 'A':
+ default:
+ Info.Slots[I].Mask = CoExecMask::All;
+ break;
+ }
+ }
+ return Info;
+}
+
+/// Get co-execution info for a WMMA instruction, selecting the per-cycle slot
+/// pattern from the opcode (and operand formats for the F8F6F4 variants).
+inline CoExecInfo getCoExecInfo(const MachineInstr &MI,
+ const SIInstrInfo &TII) {
+ unsigned Opc = MI.getOpcode();
+
+ // Scaled variants (LD_SCALE rule) absorb the next WMMA in the last I slot.
+ bool HasScaling = AMDGPU::getHasMatrixScale(Opc);
+
+ // The F8F6F4 family is the only WMMA carrying matrix format operands, and its
+ // window depends on them: both inputs f4 issue in 4 cycles, anything wider in
+ // 8. This matches the PredIsNotBothF4_WMMA_SCALE latency variant.
+ if (const MachineOperand *FmtA =
+ TII.getNamedOperand(MI, AMDGPU::OpName::matrix_a_fmt)) {
+ const MachineOperand *FmtB =
+ TII.getNamedOperand(MI, AMDGPU::OpName::matrix_b_fmt);
+ bool BothF4 = FmtB && FmtA->getImm() == AMDGPU::WMMA::MATRIX_FMT_FP4 &&
+ FmtB->getImm() == AMDGPU::WMMA::MATRIX_FMT_FP4;
+ if (BothF4)
+ return CoExecInfo::build(6, HasScaling ? "0EESVV" : "0EEIVV");
+ return CoExecInfo::build(10, HasScaling ? "0EEIEEISVV" : "0EEIEEIIVV");
+ }
+
+ switch (Opc) {
+ // 16x16x64 IU8: 16-cycle occupancy, 17-cycle window.
+ case AMDGPU::V_WMMA_I32_16X16X64_IU8_w32_threeaddr:
+ case AMDGPU::V_WMMA_I32_16X16X64_IU8_w32_twoaddr:
+ return CoExecInfo::build(17, "0EIIEEIIEEIIEEIIV");
+
+ // 16x16x64 FP8/BF8: 4-cycle occupancy, 6-cycle window.
+ case AMDGPU::V_WMMA_F16_16X16X64_BF8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_BF8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_BF8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_BF8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_FP8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_FP8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_FP8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_BF8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_BF8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_BF8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_BF8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_FP8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_FP8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_FP8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr:
+ return CoExecInfo::build(6, "0EEIVV");
+
+ // 16x16x32 F16/BF16: 8-cycle occupancy, 9-cycle window.
+ case AMDGPU::V_SWMMAC_BF16_16X16X32_BF16_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_BF16_16X16X32_BF16_w64_twoaddr:
+ case AMDGPU::V_SWMMAC_F16_16X16X32_F16_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F16_16X16X32_F16_w64_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X32_BF16_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X32_BF16_w64_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X32_F16_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X32_F16_w64_twoaddr:
+ case AMDGPU::V_WMMA_BF16F32_16X16X32_BF16_w32_threeaddr:
+ case AMDGPU::V_WMMA_BF16F32_16X16X32_BF16_w32_twoaddr:
+ case AMDGPU::V_WMMA_BF16_16X16X32_BF16_w32_threeaddr:
+ case AMDGPU::V_WMMA_BF16_16X16X32_BF16_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X32_F16_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X32_F16_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X32_BF16_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X32_BF16_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X32_F16_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X32_F16_w32_twoaddr:
+ return CoExecInfo::build(9, "0EIIEEIIV");
+
+ // 16x16x128 FP8/BF8: 8-cycle occupancy, 10-cycle window.
+ case AMDGPU::V_SWMMAC_F16_16X16X128_BF8_BF8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F16_16X16X128_BF8_FP8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F16_16X16X128_FP8_BF8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F16_16X16X128_FP8_FP8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X128_BF8_BF8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X128_BF8_FP8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X128_FP8_BF8_w32_twoaddr:
+ case AMDGPU::V_SWMMAC_F32_16X16X128_FP8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_BF8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_BF8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_BF8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_BF8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_FP8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_FP8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_FP8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F16_16X16X128_FP8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_BF8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_BF8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_BF8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_BF8_FP8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_FP8_BF8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_FP8_BF8_w32_twoaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_FP8_FP8_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr:
+ return CoExecInfo::build(10, "0EEIEEIIVV");
+
+ // 32x16x128 F4: 8-cycle occupancy, 10-cycle window.
+ case AMDGPU::V_WMMA_F32_32X16X128_F4_w32_threeaddr:
+ case AMDGPU::V_WMMA_F32_32X16X128_F4_w32_twoaddr:
+ case AMDGPU::V_WMMA_SCALE16_F32_32X16X128_F4_w32_threeaddr:
+ case AMDGPU::V_WMMA_SCALE16_F32_32X16X128_F4_w32_twoaddr:
+ case AMDGPU::V_WMMA_SCALE_F32_32X16X128_F4_w32_threeaddr:
+ case AMDGPU::V_WMMA_SCALE_F32_32X16X128_F4_w32_twoaddr:
+ return CoExecInfo::build(10, HasScaling ? "0EEIEIESVV" : "0EEIEIEIVV");
+
+ default:
+ // Permissive window for variants without a modeled slot pattern.
+ return CoExecInfo::build(9, "AAAAAAAAA");
+ }
+}
+
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 504c3aae6ca57..726be8c7f0982 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -13,6 +13,7 @@
#include "AMDGPUCoExecSchedStrategy.h"
#include "AMDGPUIGroupLP.h"
+#include "GCNHazardRecognizer.h"
#include "llvm/Support/Debug.h"
using namespace llvm;
@@ -68,6 +69,9 @@ InstructionFlavor llvm::AMDGPU::classifyFlavor(const MachineInstr &MI,
if (SII.isVALU(MI, /*AllowLDSDMA=*/true))
return InstructionFlavor::SingleCycleVALU;
+ if (SII.isSMRD(MI))
+ return InstructionFlavor::SMEM;
+
if (SII.isDS(MI))
return InstructionFlavor::DS;
@@ -434,6 +438,14 @@ void AMDGPUCoExecSchedStrategy::initialize(ScheduleDAGMI *DAG) {
GCNSchedStrategy::initialize(DAG);
Heurs.initialize(DAG, SchedModel, TRI);
+
+ // Replace the default hazard recognizer with our PreRA one so that pre-RA
+ // scheduling accounts for WMMA co-execution slot constraints. This must
+ // happen after GCNSchedStrategy::initialize() because
+ // GenericScheduler::initialize() calls SchedBoundary::reset(), which deletes
+ // and recreates the hazard recognizer each region.
+ Top.HazardRec = std::make_unique<GCNHazardRecognizer>(
+ DAG->MF, GCNHazardRecognizer::OperatingMode::PreRA);
}
void AMDGPUCoExecSchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
index 3894a08ee90d0..4ea6853e46a64 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.h
@@ -14,6 +14,7 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECSCHEDSTRATEGY_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECSCHEDSTRATEGY_H
+#include "AMDGPUCoExecInfo.h"
#include "GCNSchedStrategy.h"
#include "llvm/CodeGen/MachineScheduler.h"
@@ -21,104 +22,6 @@ namespace llvm {
namespace AMDGPU {
-//===----------------------------------------------------------------------===//
-// Instruction Flavor Classification
-//===----------------------------------------------------------------------===//
-
-enum class InstructionFlavor : uint8_t {
- WMMA, // WMMA/MFMA matrix operations
- SingleCycleVALU, // Single-cycle VALU (not TRANS32, not multi-cycle CVT)
- TRANS, // Transcendental ops (v_exp, v_log, etc.)
- MultiCycleVALU, // VALU instructions with repeat rate > 1
- VMEM, // FLAT/GLOBAL memory operations
- DS, // LDS/GDS operations
- SALU, // Scalar ALU
- DMA, // Tensor DMA operations
- Fence, // Fences and waits
- Other, // Everything else
- NUM_FLAVORS
-};
-
-inline StringRef getFlavorName(InstructionFlavor F) {
- switch (F) {
- case InstructionFlavor::WMMA:
- return "WMMA";
- case InstructionFlavor::SingleCycleVALU:
- return "VALU(1c)";
- case InstructionFlavor::TRANS:
- return "TRANS";
- case InstructionFlavor::MultiCycleVALU:
- return "VALU(Nc)";
- case InstructionFlavor::VMEM:
- return "VMEM";
- case InstructionFlavor::DS:
- return "DS";
- case InstructionFlavor::SALU:
- return "SALU";
- case InstructionFlavor::DMA:
- return "DMA";
- case InstructionFlavor::Fence:
- return "Fence";
- case InstructionFlavor::Other:
- return "Other";
- case InstructionFlavor::NUM_FLAVORS:
- llvm_unreachable("Unknown InstructionFlavor");
- }
- llvm_unreachable("Unknown InstructionFlavor");
-}
-
-inline StringRef getFlavorShortName(InstructionFlavor F) {
- switch (F) {
- case InstructionFlavor::WMMA:
- return "W";
- case InstructionFlavor::SingleCycleVALU:
- return "V";
- case InstructionFlavor::TRANS:
- return "T";
- case InstructionFlavor::MultiCycleVALU:
- return "C";
- case InstructionFlavor::VMEM:
- return "M";
- case InstructionFlavor::DS:
- return "D";
- case InstructionFlavor::SALU:
- return "S";
- case InstructionFlavor::DMA:
- return "X";
- case InstructionFlavor::Fence:
- return "F";
- case InstructionFlavor::Other:
- return "O";
- case InstructionFlavor::NUM_FLAVORS:
- llvm_unreachable("Unknown InstructionFlavor");
- }
- llvm_unreachable("Unknown InstructionFlavor");
-}
-
-InstructionFlavor classifyFlavor(const MachineInstr &MI,
- const SIInstrInfo &SII);
-
-using FlavorGroup = SmallVector<InstructionFlavor, 4>;
-
-namespace FlavorGroups {
-inline FlavorGroup allVALU() {
- return {InstructionFlavor::SingleCycleVALU, InstructionFlavor::TRANS,
- InstructionFlavor::MultiCycleVALU};
-}
-inline FlavorGroup allMem() {
- return {InstructionFlavor::VMEM, InstructionFlavor::DS,
- InstructionFlavor::DMA};
-}
-inline FlavorGroup individual(InstructionFlavor F) { return {F}; }
-inline FlavorGroup all() {
- FlavorGroup G;
- for (unsigned I = 0;
- I < static_cast<unsigned>(InstructionFlavor::NUM_FLAVORS); ++I)
- G.push_back(static_cast<InstructionFlavor>(I));
- return G;
-}
-} // namespace FlavorGroups
-
/// AMDGPU-specific scheduling decision reasons. These provide more granularity
/// than the generic CandReason enum for debugging purposes.
enum class AMDGPUSchedReason : uint8_t {
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index af326d60aacb4..a3842f43a62fa 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -26,6 +26,9 @@
using namespace llvm;
#define DEBUG_TYPE "gcn-hazard-recognizer"
+// Opt-in debug type for the per-candidate co-execution slot traces, which are
+// far too noisy for the normal debug output. Pass both types to get everything.
+#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
STATISTIC(NumWMMANopsHoisted,
"Number of WMMA hazard V_NOPs hoisted from loops");
@@ -71,20 +74,371 @@ static cl::opt<bool> EnableWMMAVnopHoisting(
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
const GCNSubtarget &ST);
-GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
- MachineLoopInfo *MLI)
- : IsHazardRecognizerMode(false), CurrCycleInstr(nullptr), MF(MF),
+GCNHazardRecognizer::GCNHazardRecognizer(
+ const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
+ MachineLoopInfo *MLI)
+ : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
+ LLVM_DEBUG({
+ if (isPreRA())
+ dbgs() << " PreRA hazard recognizer: " << MF.getName() << "\n";
+ });
+}
+
+GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
+ MachineLoopInfo *MLI)
+ : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
+
+GCNHazardRecognizer::~GCNHazardRecognizer() {
+ // Dump any active co-execution window that did not complete naturally
+ // (e.g. region ended before the window expired).
+ LLVM_DEBUG({
+ if (CurrentCoExecStage.has_value()) {
+ unsigned Stage = *CurrentCoExecStage;
+ if (Stage < AMDGPU::MaxCoExecStages)
+ CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
+ dbgs() << " CoExec window ended at stage " << Stage << ":\n";
+ dumpCoExecWindow();
+ }
+ });
}
void GCNHazardRecognizer::Reset() {
EmittedInstrs.clear();
EmittedVALUInstrs.clear();
HasPendingWMMACoexecHazard = false;
+ if (isSchedulerMode())
+ schedulerReset();
+}
+
+void GCNHazardRecognizer::schedulerReset() {
+ LLVM_DEBUG({
+ if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
+ CyclesUntilVALU > 0)
+ dbgs() << " Scheduler Reset: clearing co-exec window, TRANS="
+ << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
+ });
+ CurrentCoExecStage = std::nullopt;
+ CoExecWindowStartCycle = 0;
+ CyclesUntilTRANS = 0;
+ CyclesUntilVALU = 0;
+ ActiveCoExecInfo = AMDGPU::CoExecInfo();
+ CoExecWindowLog.fill('.');
+}
+
+void GCNHazardRecognizer::dumpCoExecWindow() const {
+ unsigned W = ActiveCoExecInfo.TotalWindow;
+ if (W == 0)
+ return;
+
+ // Print the stage numbers row.
+ dbgs() << " Stages: ";
+ for (unsigned I = 0; I < W; ++I)
+ dbgs() << I % 10 << ' ';
+ dbgs() << '\n';
+
+ // Print the pattern row.
+ dbgs() << " Slots: ";
+ for (unsigned I = 0; I < W; ++I)
+ dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
+ dbgs() << '\n';
+
+ // Print the scheduled row.
+ dbgs() << " Scheduled: ";
+ for (unsigned I = 0; I < W; ++I)
+ dbgs() << CoExecWindowLog[I] << ' ';
+ dbgs() << '\n';
+}
+
+void GCNHazardRecognizer::schedulerAdvanceCycle() {
+ // Record what happened at the current stage of the co-exec window.
+ if (CurrentCoExecStage.has_value()) {
+ unsigned Stage = *CurrentCoExecStage;
+ if (Stage < AMDGPU::MaxCoExecStages) {
+ if (CurrCycleInstr)
+ CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
+ else
+ CoExecWindowLog[Stage] = '-';
+ }
+ }
+
+ LLVM_DEBUG({
+ bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
+ CyclesUntilVALU > 0;
+ if (HasState) {
+ dbgs() << " Scheduler AdvanceCycle:";
+ if (CurrentCoExecStage.has_value()) {
+ unsigned Stage = *CurrentCoExecStage;
+ unsigned Next = Stage + 1;
+ if (Next >= ActiveCoExecInfo.TotalWindow)
+ dbgs() << " stage " << Stage << "->expired";
+ else
+ dbgs() << " stage " << Stage << "->" << Next;
+ }
+ if (CyclesUntilTRANS > 0)
+ dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
+ << (CyclesUntilTRANS - 1);
+ if (CyclesUntilVALU > 0)
+ dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
+ dbgs() << "\n";
+ }
+ });
+
+ // Decrement hazard counters.
+ if (CyclesUntilTRANS > 0)
+ --CyclesUntilTRANS;
+ if (CyclesUntilVALU > 0)
+ --CyclesUntilVALU;
+
+ // Advance WMMA co-execution window.
+ if (CurrentCoExecStage.has_value()) {
+ unsigned Stage = *CurrentCoExecStage + 1;
+ if (Stage >= ActiveCoExecInfo.TotalWindow) {
+ // Window expired.
+ LLVM_DEBUG({
+ dbgs() << " CoExec window complete:\n";
+ dumpCoExecWindow();
+ });
+ CurrentCoExecStage = std::nullopt;
+ } else {
+ CurrentCoExecStage = Stage;
+ }
+ }
+}
+
+bool GCNHazardRecognizer::hasCoExecWindowModel() const {
+ // The co-execution slot patterns returned by getCoExecInfo() are derived from
+ // gfx1250 timings, so the window model is restricted to gfx1250 for now.
+ // gfx1251 and gfx12.5-generic report the same co-execution hazard features
+ // but have different WMMA latencies, so they need their own slot patterns
+ // before they can be modeled here.
+ return ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
+ AMDGPU::isGFX1250(ST);
+}
+
+void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
+ if (!hasCoExecWindowModel())
+ return;
+
+ // Check if this is a WMMA instruction.
+ if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI))
+ return;
+
+ // If a previous window was still active, dump it before starting a new one.
+ // Record the current stage (filled by this new WMMA) before dumping.
+ LLVM_DEBUG({
+ if (CurrentCoExecStage.has_value()) {
+ unsigned Stage = *CurrentCoExecStage;
+ if (Stage < AMDGPU::MaxCoExecStages)
+ CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
+ dbgs() << " CoExec window interrupted at stage " << Stage << ":\n";
+ dumpCoExecWindow();
+ }
+ });
+
+ // Start a new co-execution window.
+ ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
+ CurrentCoExecStage = 0;
+ CoExecWindowLog.fill('.');
+
+ LLVM_DEBUG(dbgs() << " WMMA window started: " << ActiveCoExecInfo.Pattern
+ << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
+ << " " << MI);
+}
+
+void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
+ if (!hasCoExecWindowModel())
+ return;
+ if (!SIInstrInfo::isTRANS(MI))
+ return;
+
+ // Back-to-back TRANS instructions have a 1-cycle hazard.
+ // This is checked via checkTRANSHazard() and does not create a co-exec
+ // window. The TRANS shadow slot allows anything except TRANS and
+ // multi-cycle VALU.
+ // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
+ // by 1 via AdvanceCycle) before the next instruction's hazard check, so
+ // the counter is observed at 1 there. That 1-cycle stall lets the
+ // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
+ // shadow slot.
+ CyclesUntilTRANS = 2;
+ LLVM_DEBUG(dbgs() << " TRANS hazard set: CyclesUntilTRANS=2\n");
+}
+
+void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
+ if (!hasCoExecWindowModel())
+ return;
+ // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
+ if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
+ return;
+
+ // Skip WMMA and TRANS - they have their own tracking.
+ if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
+ SIInstrInfo::isTRANS(MI))
+ return;
+
+ unsigned RepeatRate = TII.getRepeatRate(MI);
+ if (RepeatRate > 1) {
+ // bumpCycle's AdvanceCycle decrements once before the next pick's
+ // hazard check (same convention as CyclesUntilTRANS), so to expose
+ // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
+ CyclesUntilVALU = RepeatRate;
+ LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate
+ << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
+ }
+}
+
+AMDGPU::CoExecMaskT
+GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
+ const SIInstrInfo &TII) {
+ return AMDGPU::getCoExecMask(AMDGPU::classifyFlavor(MI, TII));
+}
+
+unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
+ if (!CyclesUntilTRANS)
+ return 0;
+
+ // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
+ if (SIInstrInfo::isTRANS(MI))
+ return CyclesUntilTRANS;
+
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
+ TII.getRepeatRate(MI) > 1)
+ return CyclesUntilTRANS;
+
+ return 0;
+}
+
+unsigned
+GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
+ if (!CyclesUntilVALU)
+ return 0;
+
+ // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
+ // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
+ // SALU, control) can fill the shadow.
+ if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
+ !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
+ !SIInstrInfo::isTRANS(MI))
+ return 0;
+
+ return CyclesUntilVALU;
+}
+
+unsigned
+GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
+ // No hazard if not in a WMMA window.
+ if (!CurrentCoExecStage.has_value())
+ return 0;
+
+ unsigned Stage = *CurrentCoExecStage;
+ AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
+ // Check if the instruction can co-execute at the current stage.
+ if (ActiveCoExecInfo.canCoExec(InstMask, Stage))
+ return 0;
+
+ // Find next allowed stage and return stall cycles.
+ auto NextStage = ActiveCoExecInfo.findNextAllowedStage(InstMask, Stage);
+ if (NextStage.has_value()) {
+ unsigned StallCycles = *NextStage - Stage;
+ DEBUG_WITH_TYPE(
+ DEBUG_TYPE_VERBOSE,
+ dbgs() << " CoExec stall: stage=" << Stage << "("
+ << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
+ << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
+ << " -> stall " << StallCycles << " (next allowed=" << *NextStage
+ << ")\n"
+ << " " << MI);
+ return StallCycles;
+ }
+
+ // No compatible slot in window - stall until window ends.
+ unsigned StallCycles = ActiveCoExecInfo.TotalWindow - Stage;
+ DEBUG_WITH_TYPE(
+ DEBUG_TYPE_VERBOSE,
+ dbgs() << " CoExec stall: stage=" << Stage << "("
+ << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
+ << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
+ << StallCycles << " (window ends)\n"
+ << " " << MI);
+ return StallCycles;
+}
+
+unsigned
+GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
+ // This models a VALU caught in both a WMMA and a TRANS shadow.
+ if (!hasCoExecWindowModel())
+ return 0;
+
+ // No hazard if not in a WMMA window.
+ if (!CurrentCoExecStage.has_value())
+ return 0;
+
+ if (!CyclesUntilTRANS)
+ return 0;
+
+ if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
+ SIInstrInfo::isLDSDMA(MI))
+ return 0;
+
+ // We have a VALU instruction that is under both a TRANS and WMMA shadow.
+ // We need to wait for at least one to clear.
+
+ unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
+ AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
+ // Check if the instruction can co-execute at the current stage.
+ if (ActiveCoExecInfo.canCoExec(InstMask, LookAheadStage))
+ return CyclesUntilTRANS;
+
+ // Find next allowed stage and return stall cycles.
+ auto NextStage =
+ ActiveCoExecInfo.findNextAllowedStage(InstMask, LookAheadStage);
+ if (NextStage.has_value()) {
+ unsigned StallCycles = *NextStage - *CurrentCoExecStage;
+ return StallCycles;
+ }
+
+ // No compatible slot in window - stall until window ends.
+ unsigned StallCycles = ActiveCoExecInfo.TotalWindow - *CurrentCoExecStage;
+ return StallCycles;
+}
+
+void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
+ LLVM_DEBUG({
+ bool InWindow = CurrentCoExecStage.has_value();
+ bool HasActiveState =
+ InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
+ if (HasActiveState) {
+ if (InWindow) {
+ unsigned Stage = *CurrentCoExecStage;
+ dbgs() << " Stage " << Stage << "("
+ << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
+ << ") Emit ["
+ << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
+ << "]: " << *MI;
+ } else {
+ dbgs() << " Emit ["
+ << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
+ << "]: " << *MI;
+ }
+ }
+ });
+ DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
+ bool HasActiveState = CurrentCoExecStage.has_value() ||
+ CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
+ if (!HasActiveState)
+ dbgs() << " Emit ["
+ << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
+ << "]: " << *MI;
+ });
+ updateWMMAWindowState(*MI);
+ updateTRANSState(*MI);
+ updateMultiCycleVALUState(*MI);
}
void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
@@ -93,6 +447,8 @@ void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
CurrCycleInstr = MI;
+ if (isSchedulerMode())
+ schedulerEmitInstruction(MI);
}
static bool isDivFMas(unsigned Opcode) {
@@ -194,11 +550,26 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
MachineInstr *MI = SU->getInstr();
// If we are not in "HazardRecognizerMode" and therefore not being run from
// the scheduler, track possible stalls from hazards but don't insert noops.
- auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
+ auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
if (MI->isBundle())
return NoHazard;
+ // Check co-execution slot hazards and pipeline stalls in scheduler modes.
+ if (isSchedulerMode()) {
+ if (checkMultiShadowHazard(*MI) > 0)
+ return Hazard;
+ if (checkWMMACoexecSlot(*MI) > 0)
+ return Hazard;
+ if (checkTRANSHazard(*MI) > 0)
+ return Hazard;
+ if (checkMultiCycleVALUHazard(*MI) > 0)
+ return Hazard;
+ // The remaining checks are all defined by register dependences.
+ if (!hasPhysRegs())
+ return NoHazard;
+ }
+
if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
return HazardType;
@@ -209,7 +580,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
return HazardType;
// Hazards which cannot be mitigated with S_NOPs.
- if (!IsHazardRecognizerMode) {
+ if (!isHazardRecognizerMode()) {
if (checkWMMACoexecutionHazards(MI) > 0) {
HasPendingWMMACoexecHazard = true;
return Hazard;
@@ -300,7 +671,7 @@ void GCNHazardRecognizer::processBundle() {
CurrCycleInstr = &*MI;
unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
- if (IsHazardRecognizerMode) {
+ if (isHazardRecognizerMode()) {
fixHazards(CurrCycleInstr);
insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
@@ -319,7 +690,7 @@ void GCNHazardRecognizer::processBundle() {
}
void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
- assert(IsHazardRecognizerMode);
+ assert(isHazardRecognizerMode());
unsigned NumPreNoops = PreEmitNoops(MI);
EmitNoops(NumPreNoops);
@@ -333,7 +704,7 @@ void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
}
unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
- IsHazardRecognizerMode = true;
+ assert(isHazardRecognizerMode());
CurrCycleInstr = MI;
unsigned W = PreEmitNoopsCommon(MI);
fixHazards(MI);
@@ -342,7 +713,20 @@ unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
}
unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
- return this->PreEmitNoopsCommon(MI);
+ unsigned W = 0;
+
+ // Check co-execution slot hazards and pipeline stalls in scheduler modes.
+ if (isSchedulerMode()) {
+ W = checkWMMACoexecSlot(*MI);
+ W = std::max(W, checkTRANSHazard(*MI));
+ W = std::max(W, checkMultiCycleVALUHazard(*MI));
+ W = std::max(W, checkMultiShadowHazard(*MI));
+ // The remaining checks are all defined by register dependences.
+ if (!hasPhysRegs())
+ return W;
+ }
+
+ return std::max(W, PreEmitNoopsCommon(MI));
}
unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
@@ -422,6 +806,9 @@ void GCNHazardRecognizer::EmitNoop() {
}
void GCNHazardRecognizer::AdvanceCycle() {
+ if (isSchedulerMode())
+ schedulerAdvanceCycle();
+
// When the scheduler detects a stall, it will call AdvanceCycle() without
// emitting any instructions.
if (!CurrCycleInstr) {
@@ -482,7 +869,7 @@ void GCNHazardRecognizer::AdvanceCycle() {
}
void GCNHazardRecognizer::RecedeCycle() {
- assert(!IsHazardRecognizerMode &&
+ assert(!isHazardRecognizerMode() &&
"Bottom-up scheduling shouldn't run in hazard recognizer mode");
}
@@ -633,7 +1020,7 @@ getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
int GCNHazardRecognizer::getWaitStatesSince(
IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
- if (IsHazardRecognizerMode) {
+ if (isHazardRecognizerMode()) {
auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
return WaitStates >= Limit;
};
@@ -665,7 +1052,7 @@ int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
int GCNHazardRecognizer::getWaitStatesSinceVALU(IsHazardFn IsHazard,
int Limit) const {
- if (IsHazardRecognizerMode) {
+ if (isHazardRecognizerMode()) {
auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
};
@@ -3837,7 +4224,7 @@ bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
// No reason to check this in pre-RA scheduling, SGPRs have to be allocated
// for hazard to trigger.
- if (!IsHazardRecognizerMode)
+ if (!isHazardRecognizerMode())
return false;
const SIRegisterInfo *TRI = ST.getRegisterInfo();
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index 129fd6ca13419..dc8b6b3e23aec 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -13,12 +13,15 @@
#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
+#include "AMDGPUCoExecInfo.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
#include "llvm/CodeGen/TargetSchedule.h"
+#include <array>
#include <list>
+#include <optional>
namespace llvm {
@@ -36,9 +39,22 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn;
+ /// Operating mode for the hazard recognizer. Two independent properties
+ /// follow from it.
+ ///
+ /// The scheduler modes (PreRA, PostRA) report a hazard so that the scheduler
+ /// can pick some other instruction, and to do that they track pipeline state
+ /// across the cycles they are told about. HazardRecognizerMode inserts s_nop
+ /// or v_nop to ensure correctness of the generated code.
+ ///
+ /// Operands are physical registers in every mode but PreRA, so hazards
+ /// defined by register dependences are checked only when hasPhysRegs()
+ /// holds.
+ enum class OperatingMode { PreRA, PostRA, HazardRecognizerMode };
+
private:
- // Distinguish if we are called from scheduler or hazard recognizer
- bool IsHazardRecognizerMode;
+ // Operating mode determines which hazards are checked.
+ OperatingMode Mode;
// This variable stores the instruction that has been emitted this cycle. It
// will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
@@ -71,6 +87,70 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
bool RunLdsBranchVmemWARHazardFixup;
+ //===--------------------------------------------------------------------===//
+ // WMMA Co-execution Window State
+ //===--------------------------------------------------------------------===//
+
+ /// Active WMMA co-execution info (slot masks, preferences).
+ AMDGPU::CoExecInfo ActiveCoExecInfo;
+
+ /// Current stage within the WMMA co-execution window (0-based).
+ /// nullopt when not in a WMMA window.
+ std::optional<unsigned> CurrentCoExecStage;
+
+ /// Cycle when the current WMMA window started.
+ unsigned CoExecWindowStartCycle = 0;
+
+ /// Tracks cycles until TRANS can be issued again (back-to-back TRANS hazard).
+ unsigned CyclesUntilTRANS = 0;
+
+ /// Tracks cycles until next VALU after multi-cycle VALU (CVT hazard).
+ unsigned CyclesUntilVALU = 0;
+
+ /// Debug: log of what was scheduled at each stage of the co-exec window.
+ /// '.' = not yet reached, '-' = stall, else CoExecMask short char.
+ std::array<char, AMDGPU::MaxCoExecStages> CoExecWindowLog;
+
+ /// Debug: print the co-exec window visual summary.
+ void dumpCoExecWindow() const;
+
+ /// Returns true if the co-execution window model applies to this subtarget.
+ bool hasCoExecWindowModel() const;
+
+ /// Check WMMA co-execution slot hazard.
+ /// Returns stall cycles needed before MI can be issued in the current slot.
+ unsigned checkWMMACoexecSlot(const MachineInstr &MI) const;
+
+ /// Check TRANS-after-TRANS hazard. Returns stall cycles if MI is TRANS
+ /// or multi-cycle VALU and a previous TRANS shadow is still active.
+ unsigned checkTRANSHazard(const MachineInstr &MI) const;
+
+ /// Check multi-cycle VALU hazard. Returns stall cycles if MI is a VALU
+ /// that would conflict with an active multi-cycle VALU pipeline.
+ unsigned checkMultiCycleVALUHazard(const MachineInstr &MI) const;
+
+ /// Check if we have both a TRANS and WMMA window active. If so, for VALU
+ /// instructions, return the number of stall cycles until one shadow clears.
+ unsigned checkMultiShadowHazard(const MachineInstr &MI) const;
+
+ /// Update WMMA window state when a WMMA instruction is emitted.
+ void updateWMMAWindowState(const MachineInstr &MI);
+
+ /// Update TRANS state when an instruction is emitted.
+ void updateTRANSState(const MachineInstr &MI);
+
+ /// Update multi-cycle VALU state when an instruction is emitted.
+ void updateMultiCycleVALUState(const MachineInstr &MI);
+
+ /// Scheduler-mode part of EmitInstruction().
+ void schedulerEmitInstruction(MachineInstr *MI);
+
+ /// Scheduler-mode part of AdvanceCycle().
+ void schedulerAdvanceCycle();
+
+ /// Scheduler-mode part of Reset().
+ void schedulerReset();
+
/// RegUnits of uses in the current soft memory clause.
mutable BitVector ClauseUses;
@@ -182,8 +262,57 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
int checkPermlaneHazards(MachineInstr *MI) const;
public:
+ /// Construct with explicit operating mode.
+ GCNHazardRecognizer(const MachineFunction &MF, OperatingMode Mode,
+ MachineLoopInfo *MLI = nullptr);
+
+ /// Legacy constructor - defaults to PostRA mode.
GCNHazardRecognizer(const MachineFunction &MF,
MachineLoopInfo *MLI = nullptr);
+
+ ~GCNHazardRecognizer();
+
+ /// Returns the current operating mode.
+ OperatingMode getOperatingMode() const { return Mode; }
+
+ /// Returns true if running in pre-RA scheduling mode.
+ bool isPreRA() const { return Mode == OperatingMode::PreRA; }
+
+ /// Returns true if running in post-RA scheduling mode.
+ bool isPostRA() const { return Mode == OperatingMode::PostRA; }
+
+ /// Returns true if running as a scheduler (pre-RA or post-RA).
+ bool isSchedulerMode() const { return isPreRA() || isPostRA(); }
+
+ /// Returns true if instruction operands are physical registers, so that
+ /// hazards defined by register dependences can be detected.
+ bool hasPhysRegs() const { return !isPreRA(); }
+
+ /// Returns true if running as the standalone hazard recognizer pass.
+ bool isHazardRecognizerMode() const {
+ return Mode == OperatingMode::HazardRecognizerMode;
+ }
+
+ //===--------------------------------------------------------------------===//
+ // Co-execution Window Queries
+ //===--------------------------------------------------------------------===//
+
+ /// Returns true if currently inside a WMMA co-execution window.
+ bool inCoExecWindow() const { return CurrentCoExecStage.has_value(); }
+
+ /// Returns the current stage within the co-execution window, or nullopt.
+ std::optional<unsigned> getCurrentCoExecStage() const {
+ return CurrentCoExecStage;
+ }
+
+ /// Returns the active co-execution info (slot masks, preferences).
+ const AMDGPU::CoExecInfo &getActiveCoExecInfo() const {
+ return ActiveCoExecInfo;
+ }
+
+ /// Get the CoExecMask for a given instruction.
+ static AMDGPU::CoExecMaskT getCoExecMaskForMI(const MachineInstr &MI,
+ const SIInstrInfo &TII);
// We can only issue one instruction per cycle.
bool atIssueLimit() const override { return true; }
void EmitInstruction(SUnit *SU) override;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 5f7012b40148b..d2cdd583120f5 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -301,11 +301,10 @@ unsigned GCNSchedStrategy::getStructuralStallCycles(SchedBoundary &Zone,
}
// Query HazardRecognizer for sequence-dependent hazard penalties.
- // AMDGPU currently installs GCNHazardRecognizer for MI scheduling only in
- // the post-RA configuration without vreg liveness.
- if (!DAG->hasVRegLiveness() && Zone.HazardRec &&
- Zone.HazardRec->isEnabled()) {
- auto *HR = static_cast<GCNHazardRecognizer *>(Zone.HazardRec);
+ // AMDGPUCoExecSchedStrategy installs a GCNHazardRecognizer in both
+ // pre-RA (PreRA mode) and post-RA configurations.
+ if (Zone.HazardRec && Zone.HazardRec->isEnabled()) {
+ auto *HR = static_cast<GCNHazardRecognizer *>(Zone.HazardRec.get());
Stall = std::max(Stall, HR->getHazardWaitStates(MI));
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2a4ab971b245b..8ab3d78d4b822 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -10172,7 +10172,8 @@ SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
ScheduleHazardRecognizer *
SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
MachineLoopInfo *MLI) const {
- return new GCNHazardRecognizer(MF, MLI);
+ return new GCNHazardRecognizer(
+ MF, GCNHazardRecognizer::OperatingMode::HazardRecognizerMode, MLI);
}
// Called during:
@@ -11605,6 +11606,24 @@ void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));
}
+unsigned SIInstrInfo::getRepeatRate(const MachineInstr &MI) const {
+ if (!SchedModel.hasInstrSchedModel())
+ return 0;
+
+ // The repeat rate is the throughput-limiting resource occupancy: the largest
+ // number of cycles any written processor resource is held.
+ const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(&MI);
+ unsigned RepeatRate = 0;
+ for (TargetSchedModel::ProcResIter
+ PI = SchedModel.getWriteProcResBegin(SCDesc),
+ PE = SchedModel.getWriteProcResEnd(SCDesc);
+ PI != PE; ++PI) {
+ RepeatRate = std::max(RepeatRate, (unsigned)PI->ReleaseAtCycle);
+ }
+
+ return RepeatRate;
+}
+
bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
if (isIGLP(*MI))
return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 491b3aa58ac15..3a38b517415ae 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1774,6 +1774,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
// This is used if an operand is a 32 bit register but needs to be aligned
// regardless.
void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
+
+ /// Get the repeat rate for a VALU instruction from the scheduling model.
+ /// Returns 1 for regular VALU, >1 for long-latency VALU (packed, F64, etc.)
+ unsigned getRepeatRate(const MachineInstr &MI) const;
};
/// \brief Returns true if a reg:subreg pair P has a TRC class
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir b/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
new file mode 100644
index 0000000000000..b4e8a86de06b8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
@@ -0,0 +1,505 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -verify-misched %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -verify-misched -debug-only=gcn-hazard-recognizer %s -o /dev/null 2>&1 | FileCheck -check-prefix=DBG %s
+
+# REQUIRES: asserts
+
+# Tests use V_WMMA_SCALE_F32_16X16X128_F8F6F4 which has pattern 0EEIEEISVV
+# (10-cycle window, 8-cycle occupancy). The 'S' slot indicates scaled WMMA
+# can issue there (LD_SCALE rule).
+#
+# Co-execution window slots:
+# Stage: 0 1 2 3 4 5 6 7 8 9
+# Slot: E0 E E I E E I S V V
+#
+# E0 (stage 0): Control only (issue cycle)
+# E (stages 1,2,4,5): MEM + SALU allowed, NO VALU/TRANS
+# I (stages 3,6): MEM + SALU + VALU + TRANS allowed
+# S (stage 7): Same as I, plus scaled WMMA can absorb here
+# V (stages 8,9): MEM + SALU + WMMA allowed, NO VALU/TRANS
+
+--- |
+ define void @wmma_ds_salu_only() #0 { ret void }
+ define void @wmma_valu_trans_only() #0 { ret void }
+ define void @wmma_valu_in_I_slots() #0 { ret void }
+ define void @wmma_trans_in_I_slots() #0 { ret void }
+ define void @wmma_chain_back_to_back() #0 { ret void }
+ define void @wmma_mixed_ds_valu_trans() #0 { ret void }
+ define void @trans_valu_interleave() #0 { ret void }
+ define void @wmma_valu_blocked_in_V_slot() #0 { ret void }
+
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
+...
+
+# DBG-LABEL: PreRA hazard recognizer: wmma_ds_salu_only
+# DBG: CoExec window complete:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 E E I E E I S V V
+---
+name: wmma_ds_salu_only
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_ds_salu_only
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF6]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_1]], [[DEF6]], implicit-def dead $scc
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 64, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_3:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_2]], [[DEF7]], implicit-def dead $scc
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[DS_READ_B128_gfx9_4]], implicit [[S_ADD_I32_3]], implicit %17
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:sreg_32 = IMPLICIT_DEF
+ %7:sreg_32 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %8:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
+ %9:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
+ %10:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
+ %11:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
+ %12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 64, 0, implicit $exec
+ %13:sreg_32 = S_ADD_I32 %6, %7, implicit-def dead $scc
+ %14:sreg_32 = S_ADD_I32 %13, %7, implicit-def dead $scc
+ %15:sreg_32 = S_ADD_I32 %14, %6, implicit-def dead $scc
+ %16:sreg_32 = S_ADD_I32 %15, %7, implicit-def dead $scc
+ early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %16, implicit %17
+...
+
+# DBG-LABEL: PreRA hazard recognizer: wmma_valu_trans_only
+# DBG: CoExec window ended at stage 7:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 - - I - - I S . .
+---
+name: wmma_valu_trans_only
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_valu_trans_only
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF5]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_EXP_F32_e32_]], implicit [[V_PK_ADD_F32_]], implicit %10
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %7:vreg_64_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %8:vgpr_32 = V_EXP_F32_e32 %5, implicit $mode, implicit $exec
+ %9:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %8, implicit %9, implicit %10
+...
+
+# DBG-LABEL: PreRA hazard recognizer: wmma_valu_in_I_slots
+# DBG: CoExec window ended at stage 8:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 E E I E E I S V .
+---
+name: wmma_valu_in_I_slots
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_valu_in_I_slots
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_1:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF7]], 8, [[DEF8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_2:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[V_PK_ADD_F32_]], implicit [[V_PK_ADD_F32_1]], implicit [[V_PK_ADD_F32_2]], implicit %16
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %7:vreg_64_align2 = IMPLICIT_DEF
+ %8:vreg_64_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %9:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
+ %10:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
+ %11:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
+ %13:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
+ %14:vreg_64_align2 = V_PK_ADD_F32 8, %7, 8, %8, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %15:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %8, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %9, implicit %10, implicit %12, implicit %13, implicit %11, implicit %14, implicit %15, implicit %16
+...
+
+# DBG-LABEL: PreRA hazard recognizer: wmma_trans_in_I_slots
+# DBG: CoExec window ended at stage 7:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 - - I - - I S . .
+---
+name: wmma_trans_in_I_slots
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_trans_in_I_slots
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF5]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_EXP_F32_e32_]], implicit [[V_PK_ADD_F32_]], implicit %10
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %7:vreg_64_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %8:vgpr_32 = V_EXP_F32_e32 %5, implicit $mode, implicit $exec
+ %9:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %8, implicit %9, implicit %10
+...
+
+# DBG-LABEL: PreRA hazard recognizer: wmma_chain_back_to_back
+# DBG: WMMA window started: 0EEIEEISVV (window=10)
+# The next WMMA of the chain issues in a vacant slot, which starts a new window.
+# DBG: CoExec window interrupted at stage 8:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 - - - - - - - V .
+---
+name: wmma_chain_back_to_back
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_chain_back_to_back
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF9:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF10:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF6]], [[DEF7]], 0, [[DEF8]], [[DEF9]], [[DEF10]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DEF11:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF12:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF13:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber %15:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF11]], [[DEF12]], 0, [[DEF13]], [[DEF]], [[DEF1]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF14]], [[DEF2]], 0, [[DEF3]], [[DEF4]], [[DEF5]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %15, implicit %16, implicit %17
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_256_align2 = IMPLICIT_DEF
+ %8:vgpr_32_lo256 = IMPLICIT_DEF
+ %9:vgpr_32_lo256 = IMPLICIT_DEF
+ %10:vreg_512_align2 = IMPLICIT_DEF
+ %11:vreg_512_align2 = IMPLICIT_DEF
+ %12:vreg_256_align2 = IMPLICIT_DEF
+ %13:vgpr_32_lo256 = IMPLICIT_DEF
+ %14:vgpr_32_lo256 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ early-clobber %15:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %5, %6, 0, %7, %8, %9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %10, %11, 0, %12, %13, %14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %15, implicit %16, implicit %17
+...
+
+# DBG-LABEL: PreRA hazard recognizer: wmma_mixed_ds_valu_trans
+# DBG: CoExec window complete:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 E E I E E I S - -
+---
+name: wmma_mixed_ds_valu_trans
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_mixed_ds_valu_trans
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF10:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF11:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %21:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF10]], [[DEF11]], implicit-def dead $scc
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF8]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF10]], implicit-def dead $scc
+ ; CHECK-NEXT: [[V_EXP_F32_e32_1:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF9]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[V_PK_ADD_F32_]], implicit [[V_EXP_F32_e32_]], implicit [[V_EXP_F32_e32_1]], implicit [[S_ADD_I32_1]], implicit %21
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %7:vreg_64_align2 = IMPLICIT_DEF
+ %8:vgpr_32 = IMPLICIT_DEF
+ %9:vgpr_32 = IMPLICIT_DEF
+ %10:sreg_32 = IMPLICIT_DEF
+ %11:sreg_32 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
+ %13:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
+ %14:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
+ %15:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
+ %16:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %17:vgpr_32 = V_EXP_F32_e32 %8, implicit $mode, implicit $exec
+ %18:vgpr_32 = V_EXP_F32_e32 %9, implicit $mode, implicit $exec
+ %19:sreg_32 = S_ADD_I32 %10, %11, implicit-def dead $scc
+ %20:sreg_32 = S_ADD_I32 %19, %10, implicit-def dead $scc
+ early-clobber %21:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %20, implicit %21
+...
+
+# TRANS does not create co-exec windows. The TRANS counter enforces
+# 1-cycle gaps between TRANS instructions, interleaving them with VALU.
+# DBG-LABEL: PreRA hazard recognizer: trans_valu_interleave
+# DBG: TRANS hazard set: CyclesUntilTRANS=2
+# DBG: TRANS hazard set: CyclesUntilTRANS=2
+# DBG: TRANS hazard set: CyclesUntilTRANS=2
+# DBG: TRANS hazard set: CyclesUntilTRANS=2
+---
+name: trans_valu_interleave
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: trans_valu_interleave
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF4]], [[DEF5]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_EXP_F32_e32_1:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF1]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF5]], [[DEF6]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_EXP_F32_e32_2:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF2]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF6]], [[DEF7]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_EXP_F32_e32_3:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF3]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF7]], [[DEF4]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_EXP_F32_e32_]], implicit [[V_EXP_F32_e32_1]], implicit [[V_EXP_F32_e32_2]], implicit [[V_EXP_F32_e32_3]], implicit [[V_ADD_F32_e32_]], implicit [[V_ADD_F32_e32_1]], implicit [[V_ADD_F32_e32_2]], implicit [[V_ADD_F32_e32_3]]
+ bb.0:
+ successors: %bb.1
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %3:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:vgpr_32 = IMPLICIT_DEF
+ %7:vgpr_32 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %8:vgpr_32 = V_EXP_F32_e32 %0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_EXP_F32_e32 %1, implicit $mode, implicit $exec
+ %10:vgpr_32 = V_EXP_F32_e32 %2, implicit $mode, implicit $exec
+ %11:vgpr_32 = V_EXP_F32_e32 %3, implicit $mode, implicit $exec
+ %12:vgpr_32 = V_ADD_F32_e32 %4, %5, implicit $mode, implicit $exec
+ %13:vgpr_32 = V_ADD_F32_e32 %5, %6, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_ADD_F32_e32 %6, %7, implicit $mode, implicit $exec
+ %15:vgpr_32 = V_ADD_F32_e32 %7, %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15
+...
+
+# Test 8: VALU blocked in V slots. Fill E and I slots with DS/VALU,
+# then extra VALU that cannot go in V slots (stages 8,9).
+# Only 2 I slots available (3,6) but 5 VALU - some VALU pushed past window.
+# DBG-LABEL: PreRA hazard recognizer: wmma_valu_blocked_in_V_slot
+# DBG: CoExec window complete:
+# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
+# DBG-NEXT: Slots: 0 E E I E E I S V V
+# DBG-NEXT: Scheduled: 0 E E I E - I S - -
+---
+name: wmma_valu_blocked_in_V_slot
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_valu_blocked_in_V_slot
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF9:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_1:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF7]], 8, [[DEF8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_2:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF8]], 8, [[DEF9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_3:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_4:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF9]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[V_PK_ADD_F32_]], implicit [[V_PK_ADD_F32_1]], implicit [[V_PK_ADD_F32_2]], implicit [[V_PK_ADD_F32_3]], implicit [[V_PK_ADD_F32_4]], implicit %19
+ bb.0:
+ successors: %bb.1
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32_lo256 = IMPLICIT_DEF
+ %5:vgpr_32 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %7:vreg_64_align2 = IMPLICIT_DEF
+ %8:vreg_64_align2 = IMPLICIT_DEF
+ %9:vreg_64_align2 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ %10:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
+ %11:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
+ %12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
+ %13:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
+ %14:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %15:vreg_64_align2 = V_PK_ADD_F32 8, %7, 8, %8, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %16:vreg_64_align2 = V_PK_ADD_F32 8, %8, 8, %9, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %17:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %9, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_64_align2 = V_PK_ADD_F32 8, %9, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %19
+...
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 6dd31014fb28c..eb3816ec1cbc0 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -255,10 +255,9 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: s_mov_b32 s6, 0
-; COEXEC-NEXT: s_clause 0x1
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
-; COEXEC-NEXT: s_load_b64 s[2:3], s[4:5], 0x10 nv
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
+; COEXEC-NEXT: s_load_b64 s[2:3], s[4:5], 0x10 nv
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
index ecea885fe4112..d3a526dfbe425 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
@@ -310,9 +310,11 @@ define <2 x bfloat> @v_neg_rsq_v2bf16(<2 x bfloat> %a) {
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v0.l, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_2)
-; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v0.h, v1.l
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_xor_b16 v0.l, 0x8000, v0.l
+; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v0.h, v1.l
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
; GFX1250-TRUE16-NEXT: v_xor_b16 v0.h, 0x8000, v0.h
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
@@ -323,9 +325,11 @@ define <2 x bfloat> @v_neg_rsq_v2bf16(<2 x bfloat> %a) {
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX1250-FAKE16-NEXT: v_rsq_bf16_e32 v0, v0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_2)
-; GFX1250-FAKE16-NEXT: v_rsq_bf16_e32 v1, v1
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-FAKE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX1250-FAKE16-NEXT: v_rsq_bf16_e32 v1, v1
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_xor_b32_e32 v1, 0x8000, v1
; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
index e7db1c17b73fc..9c8cef44641d6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
@@ -107,8 +107,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_cluster(ptr
; COEXEC-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; COEXEC-NEXT: v_mov_b32_e32 v48, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; COEXEC-NEXT: v_dual_mov_b32 v48, 0 :: v_dual_lshlrev_b32 v0, 4, v0
+; COEXEC-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; COEXEC-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; COEXEC-NEXT: v_and_b32_e32 v0, 0x3ff0, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
@@ -326,8 +327,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_interleaved(
; COEXEC-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; COEXEC-NEXT: v_mov_b32_e32 v16, 0
+; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; COEXEC-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: v_mov_b32_e32 v17, s1
diff --git a/llvm/test/CodeGen/AMDGPU/misched-into-wmma-hazard-shadow.mir b/llvm/test/CodeGen/AMDGPU/misched-into-wmma-hazard-shadow.mir
index 8e44d93f71e80..35bd00103550a 100644
--- a/llvm/test/CodeGen/AMDGPU/misched-into-wmma-hazard-shadow.mir
+++ b/llvm/test/CodeGen/AMDGPU/misched-into-wmma-hazard-shadow.mir
@@ -64,12 +64,12 @@ body: |
; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $sgpr0, $sgpr1, $sgpr2, $sgpr3
; GCN-NEXT: {{ $}}
; GCN-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; GCN-NEXT: $sgpr4 = S_MOV_B32 killed $sgpr0
; GCN-NEXT: $vgpr31 = V_MOV_B32_e32 killed $vgpr26, implicit $exec
; GCN-NEXT: $vgpr32 = V_MOV_B32_e32 killed $vgpr27, implicit $exec
- ; GCN-NEXT: $vgpr33 = V_MOV_B32_e32 killed $vgpr28, implicit $exec
- ; GCN-NEXT: $sgpr4 = S_MOV_B32 killed $sgpr0
; GCN-NEXT: $sgpr5 = S_MOV_B32 killed $sgpr1
; GCN-NEXT: $sgpr6 = S_MOV_B32 killed $sgpr2
+ ; GCN-NEXT: $vgpr33 = V_MOV_B32_e32 killed $vgpr28, implicit $exec
; GCN-NEXT: $sgpr7 = S_MOV_B32 killed $sgpr3
; GCN-NEXT: $vgpr30 = V_MOV_B32_e32 killed $vgpr25, implicit $exec
; GCN-NEXT: $vgpr29 = V_ADD_F32_e32 killed $vgpr24, killed $vgpr16, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-trans-multi-shadow-hazard.mir b/llvm/test/CodeGen/AMDGPU/wmma-trans-multi-shadow-hazard.mir
new file mode 100644
index 0000000000000..4a37ebce24a78
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wmma-trans-multi-shadow-hazard.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -verify-misched %s -o - | FileCheck -check-prefix=CHECK %s
+
+# Tests for the multi-shadow hazard: WMMA + TRANS + VALU cannot all execute
+# simultaneously. When both a WMMA and a TRANS instruction are actively
+# executing (their shadows overlap), a subsequent VALU instruction must stall
+# until at least one of the shadows clears.
+#
+# Uses V_WMMA_F32_16X16X32_BF16 which has pattern 0EIIEEIIV (9-cycle window,
+# 8-cycle occupancy).
+#
+# Co-execution window slots:
+# Stage: 0 1 2 3 4 5 6 7 8
+# Slot: E0 E I I E E I I V
+#
+# When TRANS is issued at stage 1 (E slot), it creates a 2-cycle shadow.
+# A subsequent VALU at stage 2 cannot execute because both WMMA and TRANS
+# shadows are active - must wait for TRANS shadow to clear.
+
+--- |
+ define void @wmma_trans_valu() #0 { ret void }
+
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
+...
+# Test: WMMA -> TRANS -> VALU with dependency chain
+# TRANS uses WMMA result, VALU uses TRANS result. The dependency forces order.
+# The scheduler inserts SALU and COPY between to cover the gap.
+---
+name: wmma_trans_valu
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: wmma_trans_valu
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: dead early-clobber %5:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr [[DEF]], [[DEF1]], 8, [[DEF2]], 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: SCHED_BARRIER 1
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF3]], [[DEF3]], implicit-def dead $scc
+ ; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_EXP_F32_e32 [[DEF4]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e32 [[V_EXP_F32_e32_]], [[V_EXP_F32_e32_]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[DEF4]], implicit [[V_EXP_F32_e32_]], implicit [[V_ADD_F32_e32_]], implicit [[S_ADD_I32_]]
+ bb.0:
+ successors: %bb.1
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:sreg_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ S_BRANCH %bb.1
+
+ bb.1:
+ early-clobber %5:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %2, 0, 0, 0, 0, implicit $exec
+ %6:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ SCHED_BARRIER 1
+ %7:vgpr_32 = nofpexcept V_ADD_F32_e32 %6, %6, implicit $mode, implicit $exec
+ %8:sreg_32 = S_ADD_I32 %3, %3, implicit-def dead $scc
+ S_ENDPGM 0, implicit %4, implicit %6, implicit %7, implicit %8
+...
+...
More information about the llvm-commits
mailing list