[llvm] [AMDGPU] Model WMMA co-execution windows in the scheduler for gfx1250 (PR #204077)
Alexey Sachkov via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 02:05:15 PDT 2026
================
@@ -0,0 +1,461 @@
+//===-- AMDGPUCoExecInfo.h - Co-execution info ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Shared types for co-execution modeling used by GCNHazardRecognizer and the
+/// schedulers.
+///
+/// Multi-cycle instructions (WMMA, TRANS, etc.) have execution windows where
+/// other instruction types can co-execute. For WMMA, slot patterns depend on
+/// the variant:
+///
+/// E0 (Issue): Control instructions only (s_delay_alu, s_set_vgpr_msb)
+/// E (External): Memory and SALU can co-execute, no VALU
+/// I (Internal): VALU, TRANS, memory, and SALU can all co-execute
+/// V (Vacant): Memory/SALU/next-WMMA ok, NO VALU/TRANS
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
+
+#include "SIDefines.h"
+#include "SIInstrInfo.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/StringRef.h"
+#include <cassert>
+#include <cstdint>
+#include <optional>
+
+namespace llvm {
+
+namespace AMDGPU {
+
+//===----------------------------------------------------------------------===//
+// Co-execution Bitmasks
+//===----------------------------------------------------------------------===//
+
+/// Bitmask for instruction types allowed to co-execute at a stage.
+enum class CoExecMask : uint16_t {
+ None = 0,
+ CTRL = 1 << 0, // Control: s_delay_alu, s_set_vgpr_msb
+ VALU = 1 << 1, // Vector ALU
+ TRANS = 1 << 2, // Transcendentals (V_EXP etc)
+ SALU = 1 << 3, // Scalar ALU
+ DS = 1 << 4, // LDS read/write
+ VMEM = 1 << 5, // Global memory
+ SMEM = 1 << 6, // Scalar memory
+ WMMA = 1 << 7, // Next WMMA (V stages only)
+ All = 0xFFFF,
+
+ MEM = DS | VMEM | SMEM,
+ StageE0 = CTRL, // Issue: control only
+ StageE = CTRL | SALU | MEM, // External: mem/salu
+ StageI = CTRL | SALU | MEM | VALU | TRANS, // Internal: all ALU
+ // Internal + scaled-WMMA absorb: same as StageI but the next scaled
+ // WMMA may issue here - its LD_SCALE consumes the I cycle and the matrix
+ // multiply lands in the V slot that follows. Used for the last I before
+ // V of scaled patterns.
+ StageIS = StageI | WMMA,
+ StageV = CTRL | SALU | MEM | WMMA, // Vacant: no valu/trans
+ StageTR = All & ~TRANS, // TRANS co-exec: no TRANS
+
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All)
+};
+
+using CoExecMaskT = CoExecMask;
+
+//===----------------------------------------------------------------------===//
+// Instruction Flavor Classification
+//===----------------------------------------------------------------------===//
+
+/// Classification of instructions by execution characteristics.
+/// Used for scheduling decisions and co-execution slot preferences.
+enum class InstructionFlavor : uint8_t {
+ WMMA, // WMMA/MFMA matrix operations
+ SingleCycleVALU, // Single-cycle VALU (not TRANS, not multi-cycle CVT)
+ TRANS, // Transcendental ops (v_exp, v_log, etc.)
+ MultiCycleVALU, // VALU instructions with repeat rate > 1
+ VMEM, // FLAT/GLOBAL memory operations
+ SMEM, // Scalar memory operations
+ DS, // LDS/GDS operations
+ SALU, // Scalar ALU
+ DMA, // Tensor DMA operations
+ Fence, // Fences and waits
+ Other, // Everything else
+ NUM_FLAVORS
+};
+
+constexpr StringRef getFlavorName(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return "WMMA";
+ case InstructionFlavor::SingleCycleVALU:
+ return "VALU(1c)";
+ case InstructionFlavor::TRANS:
+ return "TRANS";
+ case InstructionFlavor::MultiCycleVALU:
+ return "VALU(Nc)";
+ case InstructionFlavor::VMEM:
+ return "VMEM";
+ case InstructionFlavor::SMEM:
+ return "SMEM";
+ case InstructionFlavor::DS:
+ return "DS";
+ case InstructionFlavor::SALU:
+ return "SALU";
+ case InstructionFlavor::DMA:
+ return "DMA";
+ case InstructionFlavor::Fence:
+ return "Fence";
+ case InstructionFlavor::Other:
+ return "Other";
+ case InstructionFlavor::NUM_FLAVORS:
+ return "???";
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+/// Classify \p MI into the execution flavor that drives both the scheduler's
+/// slot preferences and the hazard recognizer's co-execution masks.
+InstructionFlavor classifyFlavor(const MachineInstr &MI,
+ const SIInstrInfo &SII);
+
+/// Map a flavor to the co-execution class it occupies in a window slot.
+constexpr CoExecMaskT getCoExecMask(InstructionFlavor F) {
+ switch (F) {
+ case InstructionFlavor::WMMA:
+ return CoExecMask::WMMA;
+ case InstructionFlavor::TRANS:
+ return CoExecMask::TRANS;
+ case InstructionFlavor::SingleCycleVALU:
+ case InstructionFlavor::MultiCycleVALU:
+ // LDS DMA and tensor DMA issue on the VALU pipe.
+ case InstructionFlavor::DMA:
+ return CoExecMask::VALU;
+ case InstructionFlavor::DS:
+ return CoExecMask::DS;
+ case InstructionFlavor::VMEM:
+ return CoExecMask::VMEM;
+ case InstructionFlavor::SMEM:
+ return CoExecMask::SMEM;
+ case InstructionFlavor::SALU:
+ // Fences are s_barrier_*/s_wait_*, which issue on the scalar pipe.
+ case InstructionFlavor::Fence:
+ return CoExecMask::SALU;
+ case InstructionFlavor::Other:
+ return CoExecMask::CTRL;
+ case InstructionFlavor::NUM_FLAVORS:
+ break;
+ }
+ llvm_unreachable("Unknown InstructionFlavor");
+}
+
+//===----------------------------------------------------------------------===//
+// Co-execution Stage Type
+//===----------------------------------------------------------------------===//
+
+/// Stage type for co-execution (for annotation/display).
+enum class CoExecStageType : uint8_t {
+ NONE = 0, // Not in co-exec window
+ E0, // Issue cycle - control only
+ E, // External - MEM/SALU allowed
+ I, // Internal - MEM/SALU/VALU allowed
+ IS, // Internal + scaled-WMMA absorb (I plus next-WMMA issue)
+ V, // Vacant - MEM/SALU/WMMA allowed, no VALU
+ TR // TRANS co-exec - everything except TRANS
+};
+
+inline const char *getStageTypeName(CoExecStageType T) {
+ switch (T) {
+ case CoExecStageType::NONE:
+ return "--";
+ case CoExecStageType::E0:
+ return "E0";
+ case CoExecStageType::E:
+ return "E";
+ case CoExecStageType::I:
+ return "I";
+ case CoExecStageType::IS:
+ return "IS";
+ case CoExecStageType::V:
+ return "V";
+ case CoExecStageType::TR:
+ return "TR";
+ }
+ llvm_unreachable("Unknown CoExecStageType");
+}
+
+/// Return a human-readable name for a CoExecMask bitmask value.
+inline const char *getCoExecMaskName(CoExecMaskT Mask) {
+ switch (Mask) {
+ case CoExecMask::CTRL:
+ return "CTRL";
+ case CoExecMask::VALU:
+ return "VALU";
+ case CoExecMask::TRANS:
+ return "TRANS";
+ case CoExecMask::SALU:
+ return "SALU";
+ case CoExecMask::DS:
+ return "DS";
+ case CoExecMask::VMEM:
+ return "VMEM";
+ case CoExecMask::SMEM:
+ return "SMEM";
+ case CoExecMask::WMMA:
+ return "WMMA";
+ default:
+ return "???";
+ }
+}
+
+/// Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
+constexpr unsigned MaxCoExecStages = 32;
+
+//===----------------------------------------------------------------------===//
+// Co-execution Slot Info
+//===----------------------------------------------------------------------===//
+
+/// Per-slot info: which instruction classes may co-execute here.
+struct CoExecSlotInfo {
+ CoExecMaskT Mask = CoExecMask::All; // What CAN execute (correctness)
+};
+
+//===----------------------------------------------------------------------===//
+// Co-execution Info
+//===----------------------------------------------------------------------===//
+
+/// Co-execution characteristics for a multi-cycle instruction.
+struct CoExecInfo {
+ /// Total co-execution window size including tail.
+ unsigned TotalWindow = 0;
+ /// Per-stage slot info (capability mask).
+ CoExecSlotInfo Slots[MaxCoExecStages];
+ /// Pattern string for display (e.g., "0EIIEEIIV").
+ StringRef Pattern;
+
+ /// Default constructor - initialize to safe defaults.
+ CoExecInfo() {
+ for (unsigned I = 0; I < MaxCoExecStages; ++I)
+ Slots[I].Mask = CoExecMask::All; // Default: permissive
+ }
+
+ /// Get capability mask for a stage.
+ CoExecMaskT getMask(unsigned Stage) const {
+ return Stage < MaxCoExecStages ? Slots[Stage].Mask : CoExecMask::All;
----------------
AlexeySachkov wrote:
We assert that `TotalWindow < MaxCoExecStages` below on line 316 in `CoExecInfo::build`.
> Also should this be `TotalWindow` instead of `MaxCoExecStages`?
`CoExecSlotInfo` is default-initialized to `All`, so accessing a mask outside of `TotalWindow` is fine without any extra checks. This `if` deliberately guards against possible out-of-bounds access.
> Do we expect this API to be used when the `Stage` is outside of `MaxCoExecStages`?
`MaxCoExecStages` is rounded up from the largest WMMA window and I doubt that we would realisticaly go past that extra buffer (at least with the codebase in its current state), but I think that it is still good future-proofing in case we do distant lookaheads. Example of such lookahead can be found in `GCNHazardRecognizer::checkMultiShadowHazard` from this PR:
```
unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
// Check if the instruction can co-execute at the current stage.
if (ActiveCoExecInfo.canCoExec(InstMask, LookAheadStage))
return CyclesUntilTRANS;
```
Ok, the example isn't precise because it is about `canCoExec`, but we could theoretically ask for a co-exec mask for some upcoming slot as well.
https://github.com/llvm/llvm-project/pull/204077
More information about the llvm-commits
mailing list