[llvm] [AMDGPU] Model WMMA co-execution windows in the scheduler for gfx1250 (PR #204077)
Alexey Sachkov via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 00:46:44 PDT 2026
================
@@ -0,0 +1,480 @@
+//===-- AMDGPUCoExecInfo.h - Co-execution info ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Shared types for co-execution modeling used by GCNHazardRecognizer and the
+/// schedulers.
+///
+/// Multi-cycle instructions (WMMA, TRANS, etc.) have execution windows where
+/// other instruction types can co-execute. For WMMA, slot patterns depend on
+/// the variant:
+///
+/// E0 (Issue): Control instructions only (s_delay_alu, s_set_vgpr_msb)
+/// E (External): Memory and SALU can co-execute, no VALU
+/// I (Internal): VALU, TRANS, memory, and SALU can all co-execute
+/// V (Vacant): Memory/SALU/next-WMMA ok, NO VALU/TRANS
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
+
+#include "SIDefines.h"
+#include "SIInstrInfo.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/StringRef.h"
+#include <cstdint>
+#include <optional>
+
+namespace llvm {
+
+namespace AMDGPU {
+
+//===----------------------------------------------------------------------===//
+// Co-execution Bitmasks
+//===----------------------------------------------------------------------===//
+
+/// Bitmask for instruction types allowed to co-execute at a stage.
+enum class CoExecMask : uint16_t {
+ None = 0,
+ CTRL = 1 << 0, // Control: s_delay_alu, s_set_vgpr_msb
+ VALU = 1 << 1, // Vector ALU
+ TRANS = 1 << 2, // Transcendentals (V_EXP etc)
+ SALU = 1 << 3, // Scalar ALU
+ DS = 1 << 4, // LDS read/write
+ VMEM = 1 << 5, // Global memory
+ SMEM = 1 << 6, // Scalar memory
+ WMMA = 1 << 7, // Next WMMA (V stages only)
+ All = 0xFFFF,
+
+ MEM = DS | VMEM | SMEM,
+ StageE0 = CTRL, // Issue: control only
+ StageE = CTRL | SALU | MEM, // External: mem/salu
+ StageI = CTRL | SALU | MEM | VALU | TRANS, // Internal: all ALU
+ // Internal + scaled-WMMA absorb: same as StageI but the next scaled
+ // WMMA may issue here - its LD_SCALE consumes the I cycle and the matrix
+ // multiply lands in the V slot that follows. Used for the last I before
+ // V of scaled patterns.
+ StageIS = StageI | WMMA,
+ StageV = CTRL | SALU | MEM | WMMA, // Vacant: no valu/trans
+ StageTR = All & ~TRANS, // TRANS co-exec: no TRANS
+
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All)
+};
+
+using CoExecMaskT = CoExecMask;
+
+//===----------------------------------------------------------------------===//
+// Instruction Flavor Classification
+//===----------------------------------------------------------------------===//
+
+/// Classification of instructions by execution characteristics.
+/// Used for scheduling decisions and co-execution slot preferences.
+enum class InstructionFlavor : uint8_t {
+ WMMA, // WMMA/MFMA matrix operations
+ SingleCycleVALU, // Single-cycle VALU (not TRANS, not multi-cycle CVT)
+ TRANS, // Transcendental ops (v_exp, v_log, etc.)
+ MultiCycleVALU, // VALU instructions with repeat rate > 1
+ VMEM, // FLAT/GLOBAL memory operations
+ DS, // LDS/GDS operations
+ SALU, // Scalar ALU
+ DMA, // Tensor DMA operations
+ Fence, // Fences and waits
+ Other, // Everything else
+ NUM_FLAVORS
+};
+
+constexpr StringRef getFlavorName(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return "WMMA";
+ case InstructionFlavor::SingleCycleVALU:
+ return "VALU(1c)";
+ case InstructionFlavor::TRANS:
+ return "TRANS";
+ case InstructionFlavor::MultiCycleVALU:
+ return "VALU(Nc)";
+ case InstructionFlavor::VMEM:
+ return "VMEM";
+ case InstructionFlavor::DS:
+ return "DS";
+ case InstructionFlavor::SALU:
+ return "SALU";
+ case InstructionFlavor::DMA:
+ return "DMA";
+ case InstructionFlavor::Fence:
+ return "Fence";
+ case InstructionFlavor::Other:
+ return "Other";
+ case InstructionFlavor::NUM_FLAVORS:
+ return "???";
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+constexpr StringRef getFlavorShortName(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return "W";
+ case InstructionFlavor::SingleCycleVALU:
+ return "V";
+ case InstructionFlavor::TRANS:
+ return "T";
+ case InstructionFlavor::MultiCycleVALU:
+ return "C";
+ case InstructionFlavor::VMEM:
+ return "M";
+ case InstructionFlavor::DS:
+ return "D";
+ case InstructionFlavor::SALU:
+ return "S";
+ case InstructionFlavor::DMA:
+ return "X";
+ case InstructionFlavor::Fence:
+ return "F";
+ case InstructionFlavor::Other:
+ return "O";
+ case InstructionFlavor::NUM_FLAVORS:
+ return "?";
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+/// Classify \p MI into the execution flavor that drives both the scheduler's
+/// slot preferences and the hazard recognizer's co-execution masks.
+InstructionFlavor classifyFlavor(const MachineInstr &MI,
+ const SIInstrInfo &SII);
+
+/// Map a flavor to the co-execution class it occupies in a window slot.
+constexpr CoExecMaskT getCoExecMask(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return CoExecMask::WMMA;
+ case InstructionFlavor::TRANS:
+ return CoExecMask::TRANS;
+ case InstructionFlavor::SingleCycleVALU:
+ case InstructionFlavor::MultiCycleVALU:
+ // LDS DMA and tensor DMA issue on the VALU pipe.
+ case InstructionFlavor::DMA:
+ return CoExecMask::VALU;
+ case InstructionFlavor::DS:
+ return CoExecMask::DS;
+ case InstructionFlavor::VMEM:
+ return CoExecMask::VMEM;
+ case InstructionFlavor::SALU:
+ // Fences are s_barrier_*/s_wait_*, which issue on the scalar pipe.
+ case InstructionFlavor::Fence:
+ return CoExecMask::SALU;
+ case InstructionFlavor::Other:
+ return CoExecMask::CTRL;
+ case InstructionFlavor::NUM_FLAVORS:
+ break;
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+//===----------------------------------------------------------------------===//
+// Co-execution Stage Type
+//===----------------------------------------------------------------------===//
+
+/// Stage type for co-execution (for annotation/display).
+enum class CoExecStageType : uint8_t {
+ NONE = 0, // Not in co-exec window
+ E0, // Issue cycle - control only
+ E, // External - MEM/SALU allowed
+ I, // Internal - MEM/SALU/VALU allowed
+ IS, // Internal + scaled-WMMA absorb (I plus next-WMMA issue)
+ V, // Vacant - MEM/SALU/WMMA allowed, no VALU
+ TR // TRANS co-exec - everything except TRANS
+};
+
+inline const char *getStageTypeName(CoExecStageType T) {
+ switch (T) {
+ case CoExecStageType::NONE:
+ return "--";
+ case CoExecStageType::E0:
+ return "E0";
+ case CoExecStageType::E:
+ return "E";
+ case CoExecStageType::I:
+ return "I";
+ case CoExecStageType::IS:
+ return "IS";
+ case CoExecStageType::V:
+ return "V";
+ case CoExecStageType::TR:
+ return "TR";
+ }
+ llvm_unreachable("Unknown CoExecStageType");
+}
+
+/// Return a human-readable name for a CoExecMask bitmask value.
+inline const char *getCoExecMaskName(CoExecMaskT Mask) {
+ switch (Mask) {
+ case CoExecMask::CTRL:
+ return "CTRL";
+ case CoExecMask::VALU:
+ return "VALU";
+ case CoExecMask::TRANS:
+ return "TRANS";
+ case CoExecMask::SALU:
+ return "SALU";
+ case CoExecMask::DS:
+ return "DS";
+ case CoExecMask::VMEM:
+ return "VMEM";
+ case CoExecMask::SMEM:
+ return "SMEM";
+ case CoExecMask::WMMA:
+ return "WMMA";
+ default:
+ return "???";
+ }
+}
+
+/// Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
+constexpr unsigned MaxCoExecStages = 32;
+
+//===----------------------------------------------------------------------===//
+// Co-execution Slot Info
+//===----------------------------------------------------------------------===//
+
+/// Per-slot info: which instruction classes may co-execute here.
+struct CoExecSlotInfo {
+ CoExecMaskT Mask = CoExecMask::All; // What CAN execute (correctness)
+};
+
+//===----------------------------------------------------------------------===//
+// Co-execution Info
+//===----------------------------------------------------------------------===//
+
+/// Co-execution characteristics for a multi-cycle instruction.
+struct CoExecInfo {
+ /// Total co-execution window size including tail.
+ unsigned TotalWindow = 0;
+ /// Per-stage slot info (capability mask).
+ CoExecSlotInfo Slots[MaxCoExecStages];
+ /// Pattern string for display (e.g., "0EIIEEIIV").
+ StringRef Pattern;
+
+ /// Default constructor - initialize to safe defaults.
+ CoExecInfo() {
+ for (unsigned I = 0; I < MaxCoExecStages; ++I)
+ Slots[I].Mask = CoExecMask::All; // Default: permissive
+ }
+
+ /// Get capability mask for a stage.
+ CoExecMaskT getMask(unsigned Stage) const {
+ return Stage < MaxCoExecStages ? Slots[Stage].Mask : CoExecMask::All;
+ }
+
+ /// Check if an instruction class mask can co-execute at a given stage.
+ bool canCoExec(CoExecMaskT InstMask, unsigned Stage) const {
+ if (Stage >= TotalWindow)
+ return true;
+ return any(Slots[Stage].Mask & InstMask);
+ }
+
+ /// Find next stage where the instruction class is allowed.
+ std::optional<unsigned> findNextAllowedStage(CoExecMaskT InstMask,
+ unsigned FromStage) const {
+ for (unsigned I = FromStage; I < TotalWindow; ++I) {
+ if (any(Slots[I].Mask & InstMask))
+ return I;
+ }
+ return std::nullopt;
+ }
+
+ /// Get stage type from mask for display.
+ static CoExecStageType getStageType(CoExecMaskT Mask) {
+ if (Mask == CoExecMask::StageE0)
+ return CoExecStageType::E0;
+ if (Mask == CoExecMask::StageE)
+ return CoExecStageType::E;
+ if (Mask == CoExecMask::StageIS)
+ return CoExecStageType::IS;
+ if (Mask == CoExecMask::StageI)
+ return CoExecStageType::I;
+ if (Mask == CoExecMask::StageV)
+ return CoExecStageType::V;
+ if (Mask == CoExecMask::StageTR)
+ return CoExecStageType::TR;
+ // For 'All' or unknown, return based on what's allowed.
+ if (any(Mask & CoExecMask::VALU))
+ return CoExecStageType::I; // If VALU allowed, it's I-like
+ if (any(Mask & CoExecMask::WMMA))
+ return CoExecStageType::V; // If WMMA allowed (not VALU), V-like
+ return CoExecStageType::E; // Default to E
+ }
+
+ /// Get stage type for a specific stage.
+ CoExecStageType getType(unsigned Stage) const {
+ return getStageType(getMask(Stage));
+ }
+
+ /// Build a CoExecInfo from a pattern string.
+ static CoExecInfo build(unsigned TotalWindow, const char *Pattern);
+};
+
+//===----------------------------------------------------------------------===//
+// Co-execution Info Construction
+//===----------------------------------------------------------------------===//
+
+/// Build CoExecInfo from a pattern string.
+/// Pattern chars: '0'=E0, 'E'=External, 'I'=Internal, 'V'=Vacant,
+/// 'S'=Internal+ScaleWMMAAbsorb (I plus next scaled WMMA),
+/// 'T'=TRANS co-exec (all except TRANS), 'A'=Any
+inline CoExecInfo CoExecInfo::build(unsigned TotalWindow, const char *Pattern) {
+ CoExecInfo Info;
+ Info.TotalWindow = TotalWindow;
+ Info.Pattern = Pattern;
+
+ for (unsigned I = 0; I < Info.TotalWindow && Pattern[I]; ++I) {
----------------
AlexeySachkov wrote:
We could simply assert at the beginning that `strlen(Pattern) == TotalWindow)` instead of checking it for every symbol. Or do we expect to see patterns which are shorter than total window?
https://github.com/llvm/llvm-project/pull/204077
More information about the llvm-commits
mailing list