[llvm] AMDGPU: VALU data fast-forwarding needs no s_delay (PR #205481)
Reem Elkhouly via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 22:01:43 PDT 2026
https://github.com/amd-relkhoul created https://github.com/llvm/llvm-project/pull/205481
VALU can forward data for certain VALU -> VALU dependencies that pass a
value through VCC/carry, so no s_delay_alu is needed between the producer and
consumer.
This teaches AMDGPUInsertDelayAlu to recognize those pairs and skip
the delay.
>From eaa128e13df6dd498f1f4d2cb532315ef513b7e3 Mon Sep 17 00:00:00 2001
From: Reem Elkhouly <reem.elkhouly at amd.com>
Date: Wed, 24 Jun 2026 13:28:25 +0900
Subject: [PATCH] AMDGPU: VALU data fast-forwarding needs no s_delay
---
.../Target/AMDGPU/AMDGPUInsertDelayAlu.cpp | 83 +++++-
...delay-alu-skipped-with-fast-forwarding.mir | 260 ++++++++++++++++++
2 files changed, 340 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/insert-delay-alu-skipped-with-fast-forwarding.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index 35390fc71b5e7..1a91f0b1eab1c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -121,6 +121,9 @@ class AMDGPUInsertDelayAlu {
// until it completes.
uint8_t SALUCycles = 0;
+ // Set when this entry was produced by a data fast-forward producer
+ bool IsVCCFFProducer = false;
+
DelayInfo() = default;
DelayInfo(DelayType Type, unsigned Cycles) {
@@ -147,7 +150,8 @@ class AMDGPUInsertDelayAlu {
bool operator==(const DelayInfo &RHS) const {
return VALUCycles == RHS.VALUCycles && VALUNum == RHS.VALUNum &&
TRANSCycles == RHS.TRANSCycles && TRANSNum == RHS.TRANSNum &&
- TRANSNumVALU == RHS.TRANSNumVALU && SALUCycles == RHS.SALUCycles;
+ TRANSNumVALU == RHS.TRANSNumVALU && SALUCycles == RHS.SALUCycles &&
+ IsVCCFFProducer == RHS.IsVCCFFProducer;
}
bool operator!=(const DelayInfo &RHS) const { return !(*this == RHS); }
@@ -161,6 +165,7 @@ class AMDGPUInsertDelayAlu {
TRANSNum = std::min(TRANSNum, RHS.TRANSNum);
TRANSNumVALU = std::min(TRANSNumVALU, RHS.TRANSNumVALU);
SALUCycles = std::max(SALUCycles, RHS.SALUCycles);
+ IsVCCFFProducer = IsVCCFFProducer && RHS.IsVCCFFProducer;
}
// Update this DelayInfo after issuing an instruction of the specified type.
@@ -348,6 +353,60 @@ class AMDGPUInsertDelayAlu {
return (Imm & 0x780) ? nullptr : DelayAlu;
}
+ static bool isCmpX(const MachineInstr &MI) {
+ if (!MI.isCompare())
+ return false;
+
+ if (MI.definesRegister(AMDGPU::EXEC, /*TRI=*/nullptr))
+ return true;
+
+ return false;
+ }
+
+ // Fast-forward producers: instructions that write VCC/carry as an
+ // explicit output and are covered by the hardware fast-forward path.
+ // TODO: V_CMPX writes EXEC implicitly
+ static bool isFastForwardProducer(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_ADD_CO_U32_e64:
+ case AMDGPU::V_SUB_CO_U32_e64:
+ case AMDGPU::V_SUBREV_CO_U32_e64:
+ case AMDGPU::V_ADDC_U32_e64:
+ case AMDGPU::V_SUBB_U32_e64:
+ case AMDGPU::V_SUBBREV_U32_e64:
+ return true;
+ default:
+ // All V_CMP* instructions excluding V_CMPX.
+ return SIInstrInfo::isVALU(MI) && MI.isCompare() && !isCmpX(MI);
+ }
+ }
+
+ // Fast-forward consumers: instructions that read VCC/carry and
+ // are covered by the hardware fast-forward path.
+ // Note: V_DIV_FMAS reads VCC as *data* (not carry) and is explicitly
+ // excluded: hardware cannot fast-forward that path (4-cycle stall).
+ static bool isFastForwardConsumer(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ // Explicit VCC carry-in
+ case AMDGPU::V_ADDC_U32_e64:
+ case AMDGPU::V_SUBB_U32_e64:
+ case AMDGPU::V_SUBBREV_U32_e64:
+ case AMDGPU::V_CNDMASK_B32_e64:
+ // Implicit VCC carry-in
+ // Included here so the fast-forward suppression is correct once the pass
+ // is extended to track implicit uses (see TODO at line 403).
+ // TODO: include VALU that use implicit EXEC (produced by V_CMPX*) as
+ // a condition, once we track implicit uses.
+ case AMDGPU::V_ADDC_U32_e32:
+ case AMDGPU::V_SUBB_U32_e32:
+ case AMDGPU::V_SUBBREV_U32_e32:
+ case AMDGPU::V_CNDMASK_B32_e32:
+ return true;
+ default:
+ return false;
+ }
+ }
+
bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
DelayState State;
for (auto *Pred : MBB.predecessors())
@@ -400,6 +459,18 @@ class AMDGPUInsertDelayAlu {
// ignore this operand.
if (MI.getOpcode() == AMDGPU::V_WRITELANE_B32 && Op.isTied())
continue;
+ // Suppress the delay for VCC operands when both the
+ // producer and consumer are in the hardware fast-forward set.
+ Register Reg = Op.getReg();
+ if (ST->getGeneration() >= AMDGPUSubtarget::GFX13 &&
+ (Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC) &&
+ isFastForwardConsumer(MI) &&
+ llvm::all_of(TRI->regunits(Reg), [&](MCRegUnit Unit) {
+ auto It = State.find(Unit);
+ return It != State.end() && It->second.IsVCCFFProducer;
+ })) {
+ continue;
+ }
for (MCRegUnit Unit : TRI->regunits(Op.getReg())) {
auto It = State.find(Unit);
if (It != State.end()) {
@@ -429,11 +500,17 @@ class AMDGPUInsertDelayAlu {
if (Type != OTHER) {
// TODO: Scan implicit defs too?
+ bool IsFFProd = ST->getGeneration() >= AMDGPUSubtarget::GFX13 &&
+ isFastForwardProducer(MI);
for (const auto &Op : MI.defs()) {
unsigned Latency = SchedModel->computeOperandLatency(
&MI, Op.getOperandNo(), nullptr, 0);
- for (MCRegUnit Unit : TRI->regunits(Op.getReg()))
- State[Unit] = DelayInfo(Type, Latency);
+ DelayInfo Info(Type, Latency);
+ Register Reg = Op.getReg();
+ if (IsFFProd && (Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC))
+ Info.IsVCCFFProducer = true;
+ for (MCRegUnit Unit : TRI->regunits(Reg))
+ State[Unit] = Info;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-skipped-with-fast-forwarding.mir b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-skipped-with-fast-forwarding.mir
new file mode 100644
index 0000000000000..3485a60719cc2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-skipped-with-fast-forwarding.mir
@@ -0,0 +1,260 @@
+# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1310 -mattr=+wavefrontsize32 -verify-machineinstrs -start-before=amdgpu-insert-delay-alu %s -o - | FileCheck %s
+
+# GFX13 has hardware fast-forwarding for certain VALU -> VALU dependency
+# patterns that involve carry/mask registers. No s_delay_alu is needed for:
+# { V_CMP, add/sub with carry-out } -> { v_cndmask*, add/sub with carry-in }
+# Source: GFX13 Shader Programming Guide section 5.3.5.1 "VALU Data Forwarding".
+#
+# Each producer and each consumer is exercised at least once below.
+# Producers: V_ADD_CO_U32_e64, V_SUB_CO_U32_e64, V_SUBREV_CO_U32_e64,
+# V_ADD_CO_CI_U32_e64, V_SUB_CO_CI_U32_e64, V_SUBREV_CO_CI_U32_e64,
+# V_CMP* (all VOPC, exercised via V_CMP_EQ_I32_e64)
+# Consumers (explicit e64): V_ADDC_U32_e64, V_SUBB_U32_e64, V_SUBREV_CO_CI_U32_e64,
+# V_CNDMASK_B32_e64
+# Consumers (implicit e32): V_ADDC_U32_e32, V_SUBB_U32_e32, V_SUBBREV_U32_e32,
+# V_CNDMASK_B32_e32
+#
+# NOTE: The vcc_lo -> V_DIV_FMAS negative case (4-cycle stall, no fast-forward) is
+# commented out below: the pass misses that dependency because V_DIV_FMAS reads vcc
+# as an *implicit* operand and the pass only scans explicit_uses(). That is a
+# separate pre-existing gap; re-enable the test case once it is fixed.
+
+--- |
+
+ ; Case 1a: V_ADD_CO_U32_e64 (producer) -> V_ADDC_U32_e64 (consumer, explicit).
+ ; No s_delay_alu expected after fix.
+ define void @add_co_to_addc_e64() {
+ ; CHECK-LABEL: add_co_to_addc_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+ ; CHECK-NEXT: v_add_co_ci_u32_e64 v3, vcc_lo, v3, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 1b: V_SUB_CO_U32_e64 (producer) -> V_SUBB_U32_e64 (consumer, explicit).
+ ; No s_delay_alu expected after fix.
+ define void @sub_co_to_subb_e64() {
+ ; CHECK-LABEL: sub_co_to_subb_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_sub_co_u32 v2, vcc_lo, v2, v4
+ ; CHECK-NEXT: v_sub_co_ci_u32_e64 v3, vcc_lo, v3, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 1c: V_SUBREV_CO_U32_e64 (producer) -> V_ADDC_U32_e64 (consumer, explicit).
+ ; No s_delay_alu expected after fix.
+ define void @subrev_co_to_addc_e64() {
+ ; CHECK-LABEL: subrev_co_to_addc_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_subrev_co_u32 v2, vcc_lo, v2, v4
+ ; CHECK-NEXT: v_add_co_ci_u32_e64 v3, vcc_lo, v3, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 1d: V_ADDC_U32_e64 (dual producer+consumer) chained.
+ ; No s_delay_alu expected after fix (neither carry-in step nor carry-out step).
+ define void @add_co_ci_chain_e64() {
+ ; CHECK-LABEL: add_co_ci_chain_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v1
+ ; CHECK-NEXT: v_add_co_ci_u32_e64 v2, vcc_lo, v2, v3, vcc_lo
+ ; CHECK-NEXT: v_add_co_ci_u32_e64 v4, vcc_lo, v4, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 1e: V_SUBB_U32_e64 (dual producer+consumer) chained.
+ ; No s_delay_alu expected after fix.
+ define void @sub_co_ci_chain_e64() {
+ ; CHECK-LABEL: sub_co_ci_chain_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
+ ; CHECK-NEXT: v_sub_co_ci_u32_e64 v2, vcc_lo, v2, v3, vcc_lo
+ ; CHECK-NEXT: v_sub_co_ci_u32_e64 v4, vcc_lo, v4, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 1f: V_SUBBREV_U32_e64 (dual producer+consumer) chained.
+ ; No s_delay_alu expected after fix.
+ define void @subrev_co_ci_chain_e64() {
+ ; CHECK-LABEL: subrev_co_ci_chain_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v1
+ ; CHECK-NEXT: v_subrev_co_ci_u32_e64 v2, vcc_lo, v2, v3, vcc_lo
+ ; CHECK-NEXT: v_subrev_co_ci_u32_e64 v4, vcc_lo, v4, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 2a: V_CMP_EQ_I32_e64 (producer) -> V_CNDMASK_B32_e64 (consumer, explicit).
+ ; No s_delay_alu expected after fix.
+ define void @cmp_to_cndmask_b32_e64() {
+ ; CHECK-LABEL: cmp_to_cndmask_b32_e64:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_cmp_eq_i32_e64 vcc_lo, v0, v1
+ ; CHECK-NEXT: v_cndmask_b32_e64 v2, v3, v4, vcc_lo
+ ret void
+ }
+
+ ; Case 3a: V_ADD_CO_U32_e64 (producer) -> V_ADDC_U32_e32 (consumer, implicit VCC).
+ ; VCC is implicit in e32 - pass does not track it yet (TODO). No delay emitted
+ ; today and none expected after fix either; included to guard against regression
+ ; once implicit tracking is added.
+ define void @add_co_to_addc_e32() {
+ ; CHECK-LABEL: add_co_to_addc_e32:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+ ; CHECK-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 3b: V_SUB_CO_U32_e64 (producer) -> V_SUBB_U32_e32 (consumer, implicit VCC).
+ define void @sub_co_to_subb_e32() {
+ ; CHECK-LABEL: sub_co_to_subb_e32:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_sub_co_u32 v2, vcc_lo, v2, v4
+ ; CHECK-NEXT: v_sub_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 3c: V_SUB_CO_U32_e64 (producer) -> V_SUBBREV_U32_e32 (consumer, implicit VCC).
+ define void @sub_co_to_subbrev_e32() {
+ ; CHECK-LABEL: sub_co_to_subbrev_e32:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_sub_co_u32 v2, vcc_lo, v2, v4
+ ; CHECK-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo
+ ret void
+ }
+
+ ; Case 3d: V_CMP_EQ_I32_e64 (producer) -> V_CNDMASK_B32_e32 (consumer, implicit VCC).
+ define void @cmp_to_cndmask_b32_e32() {
+ ; CHECK-LABEL: cmp_to_cndmask_b32_e32:
+ ; CHECK: ; %bb.0:
+ ; CHECK-NEXT: v_cmp_eq_i32_e64 vcc_lo, v0, v1
+ ; CHECK-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+ ret void
+ }
+
+...
+
+# Case 1a: V_ADD_CO_U32_e64 -> V_ADDC_U32_e64
+---
+name: add_co_to_addc_e64
+body: |
+ bb.0:
+ $vgpr2, $vcc_lo = V_ADD_CO_U32_e64 $vgpr2, $vgpr4, 0, implicit $exec
+ $vgpr3, $vcc_lo = V_ADDC_U32_e64 $vgpr3, $vgpr5, $vcc_lo, 0, implicit $exec
+...
+
+# Case 1b: V_SUB_CO_U32_e64 -> V_SUBB_U32_e64
+---
+name: sub_co_to_subb_e64
+body: |
+ bb.0:
+ $vgpr2, $vcc_lo = V_SUB_CO_U32_e64 $vgpr2, $vgpr4, 0, implicit $exec
+ $vgpr3, $vcc_lo = V_SUBB_U32_e64 $vgpr3, $vgpr5, $vcc_lo, 0, implicit $exec
+...
+
+# Case 1c: V_SUBREV_CO_U32_e64 -> V_ADDC_U32_e64
+---
+name: subrev_co_to_addc_e64
+body: |
+ bb.0:
+ $vgpr2, $vcc_lo = V_SUBREV_CO_U32_e64 $vgpr2, $vgpr4, 0, implicit $exec
+ $vgpr3, $vcc_lo = V_ADDC_U32_e64 $vgpr3, $vgpr5, $vcc_lo, 0, implicit $exec
+...
+
+# Case 1d: V_ADDC_U32_e64 acting as both producer and consumer (3-instruction chain).
+# MIR pseudo: V_ADDC_U32_e64 (GFX13 assembly: v_add_co_ci_u32_e64)
+---
+name: add_co_ci_chain_e64
+body: |
+ bb.0:
+ $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+ $vgpr2, $vcc_lo = V_ADDC_U32_e64 $vgpr2, $vgpr3, $vcc_lo, 0, implicit $exec
+ $vgpr4, $vcc_lo = V_ADDC_U32_e64 $vgpr4, $vgpr5, $vcc_lo, 0, implicit $exec
+...
+
+# Case 1e: V_SUBB_U32_e64 acting as both producer and consumer.
+# MIR pseudo: V_SUBB_U32_e64 (GFX13 assembly: v_sub_co_ci_u32_e64)
+---
+name: sub_co_ci_chain_e64
+body: |
+ bb.0:
+ $vgpr0, $vcc_lo = V_SUB_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+ $vgpr2, $vcc_lo = V_SUBB_U32_e64 $vgpr2, $vgpr3, $vcc_lo, 0, implicit $exec
+ $vgpr4, $vcc_lo = V_SUBB_U32_e64 $vgpr4, $vgpr5, $vcc_lo, 0, implicit $exec
+...
+
+# Case 1f: V_SUBBREV_U32_e64 acting as both producer and consumer.
+# MIR pseudo: V_SUBBREV_U32_e64 (GFX13 assembly: v_subrev_co_ci_u32_e64)
+---
+name: subrev_co_ci_chain_e64
+body: |
+ bb.0:
+ $vgpr0, $vcc_lo = V_SUB_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+ $vgpr2, $vcc_lo = V_SUBBREV_U32_e64 $vgpr2, $vgpr3, $vcc_lo, 0, implicit $exec
+ $vgpr4, $vcc_lo = V_SUBBREV_U32_e64 $vgpr4, $vgpr5, $vcc_lo, 0, implicit $exec
+...
+
+# Case 2a: V_CMP_EQ_I32_e64 -> V_CNDMASK_B32_e64
+---
+name: cmp_to_cndmask_b32_e64
+body: |
+ bb.0:
+ $vcc_lo = V_CMP_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ $vgpr2 = V_CNDMASK_B32_e64 0, $vgpr3, 0, $vgpr4, $vcc_lo, implicit $exec
+...
+
+# Case 3a: V_ADD_CO_U32_e64 -> V_ADDC_U32_e32 (implicit VCC consumer)
+# Currently the pass does not track implicit uses, so the fast-forward suppression
+# for this case is preparatory (no behaviour change until the TODO is resolved).
+---
+name: add_co_to_addc_e32
+body: |
+ bb.0:
+ $vgpr2, $vcc_lo = V_ADD_CO_U32_e64 $vgpr2, $vgpr4, 0, implicit $exec
+ $vgpr3 = V_ADDC_U32_e32 $vgpr3, $vgpr5, implicit-def $vcc, implicit $vcc, implicit $exec
+...
+
+# Case 3b: V_SUB_CO_U32_e64 -> V_SUBB_U32_e32 (implicit VCC consumer)
+---
+name: sub_co_to_subb_e32
+body: |
+ bb.0:
+ $vgpr2, $vcc_lo = V_SUB_CO_U32_e64 $vgpr2, $vgpr4, 0, implicit $exec
+ $vgpr3 = V_SUBB_U32_e32 $vgpr3, $vgpr5, implicit-def $vcc, implicit $vcc, implicit $exec
+...
+
+# Case 3c: V_SUB_CO_U32_e64 -> V_SUBBREV_U32_e32 (implicit VCC consumer)
+---
+name: sub_co_to_subbrev_e32
+body: |
+ bb.0:
+ $vgpr2, $vcc_lo = V_SUB_CO_U32_e64 $vgpr2, $vgpr4, 0, implicit $exec
+ $vgpr3 = V_SUBBREV_U32_e32 $vgpr3, $vgpr5, implicit-def $vcc, implicit $vcc, implicit $exec
+...
+
+# Case 3d: V_CMP_EQ_I32_e64 -> V_CNDMASK_B32_e32 (implicit VCC consumer)
+---
+name: cmp_to_cndmask_b32_e32
+body: |
+ bb.0:
+ $vcc_lo = V_CMP_EQ_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ $vgpr2 = V_CNDMASK_B32_e32 $vgpr3, $vgpr4, implicit $vcc, implicit $exec
+...
+
+# DISABLED: Negative case (no fast-forward on any target):
+# carry-out (vcc_lo) -> V_DIV_FMAS_F32 which reads vcc as *data* (not carry).
+# Hardware specification: cannot fast-forward VCC into V_DIV_FMAS - 4-cycle stall,
+# so s_delay_alu must be emitted even on GFX13.
+# BUG: V_DIV_FMAS_F32 reads vcc as an implicit operand; the pass only scans
+# explicit_uses() and therefore never sees the dependency, so no delay is emitted.
+# Re-enable once that pre-existing bug is fixed.
+#
+# ---
+# name: carry_to_div_fmas_no_fastforward
+# body: |
+# bb.0:
+# $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, $vgpr1, 0, implicit $exec
+# $vgpr2 = V_DIV_FMAS_F32_e64 0, $vgpr2, 0, $vgpr3, 0, $vgpr4, 0, 0, implicit $mode, implicit $vcc, implicit $exec
+# ...
More information about the llvm-commits
mailing list