[llvm] [AMDGPU] Work around gfx1250 VOPD3 FP64 OPY hazard (PR #224741)
Joe Nash via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 12:46:21 PDT 2026
https://github.com/Sisyph updated https://github.com/llvm/llvm-project/pull/224741
>From 3c9a2d59e11bfc639651f11a2e2c5f57ab1ef3fd Mon Sep 17 00:00:00 2001
From: Joseph Nash <joseph.nash at amd.com>
Date: Fri, 18 Sep 2026 14:53:10 -0400
Subject: [PATCH 1/3] [AMDGPU] Work around gfx1250 VOPD3 FP64 OPY hazard
In a VOPD3 instruction whose OPX is a 64-bit operation, an OPY VGPR
source operand reads back the wrong value when it names the last VGPR
the wave owns.
A wave always owns a whole number of VGPR allocation granules, 16
registers each on gfx1250, so the last VGPR it owns is always the one
just below a granule boundary. Reject a pair whose OPY reads such a
register in checkVOPDRegConstraints, leaving the two components as
separate instructions. VOPD is only an optimization, so losing a pairing
is always safe.
The wave's actual VGPR count is deliberately not consulted. It is only
known once the
whole call graph has been seen, at emission time, and for a function
with calls it is not what MachineRegisterInfo reports.
Keying on the register
number alone rejects a superset of the hazardous pairs and cannot be
invalidated by a later pass.
The workaround is gated on a new vopd3-f64-opy-src-hazard subtarget
feature, enabled for gfx1250 only.
Assisted-by: Claude Code:claude-opus-5
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 7 +
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 42 +++
.../vopd3-f64-opy-last-vgpr-gfx1250.mir | 355 ++++++++++++++++++
3 files changed, 404 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index b0fa161645ad4..68d9aad22447a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1255,6 +1255,12 @@ defm WMMACoexecutionHazards : AMDGPUSubtargetFeature<"wmma-coexecution-hazards",
/*GenPredicate=*/0
>;
+defm VOPD3F64OPYSrcHazard : AMDGPUSubtargetFeature<"vopd3-f64-opy-src-hazard",
+ "Hazard when an OPY VGPR source of a VOPD3 with a 64-bit OPX names the last "
+ "VGPR allocated to the wave",
+ /*GenPredicate=*/0
+>;
+
defm SALUFloatInsts : AMDGPUSubtargetFeature<"salu-float",
"Has SALU floating point instructions"
>;
@@ -2458,6 +2464,7 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureRealTrue16Insts,
FeatureVMovB64Inst,
FeatureVMulU64Inst,
+ FeatureVOPD3F64OPYSrcHazard,
]>;
def FeatureISAVersion12_50_STRICT : FeatureSet<
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index 63b28bf4ad645..bcde29f9ab05f 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -105,6 +105,46 @@ static bool canMapVOP3PToVOPD(const MachineInstr &MI) {
getNamedOp(MI, AMDGPU::OpName::src2).getReg();
}
+// In a VOPD3 whose OPX is a 64-bit operation an OPY VGPR source operand reads
+// back the wrong value if it names the last VGPR the wave owns.
+//
+// The number of VGPRs a wave owns is always a whole number of allocation
+// granules, so the last VGPR it owns is always the one just below a granule
+// boundary. A source which is not there cannot be it, whatever the wave's VGPR
+// count turns out to be. That count is deliberately not consulted: it is only
+// known once the whole call graph has been seen, at emission time.
+static bool isVOPD3F64OPYSrcHazard(const SIInstrInfo &TII,
+ const MachineInstr &MIX,
+ const MachineInstr &MIY) {
+ const MachineFunction &MF = *MIX.getMF();
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!ST.hasVOPD3F64OPYSrcHazard())
+ return false;
+
+ // Every 64-bit VOPD3 OPX opcode has a 64-bit vdst, and no 32-bit one has.
+ int VDstIdx =
+ AMDGPU::getNamedOperandIdx(MIX.getOpcode(), AMDGPU::OpName::vdst);
+ assert(VDstIdx != -1 && "VOPD3 OPX component must have a vdst");
+ if (TII.getOpSize(MIX, VDstIdx) != 8)
+ return false;
+
+ // Targets with this hazard do not support dynamic VGPR allocation.
+ unsigned Granule =
+ AMDGPU::IsaInfo::getVGPRAllocGranule(ST, /*DynamicVGPRBlockSize=*/0);
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ for (AMDGPU::OpName Name :
+ {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
+ const MachineOperand *Src = TII.getNamedOperand(MIY, Name);
+ // Every OPY source which can be a VGPR is 32 bits wide.
+ if (!Src || !Src->isReg() ||
+ !AMDGPU::VGPR_32RegClass.contains(Src->getReg()))
+ continue;
+ if ((TRI->getHWRegIndex(Src->getReg()) + 1) % Granule == 0)
+ return true;
+ }
+ return false;
+}
+
static bool canMaterializeVOPDLiterals(const MachineFunction &MF) {
// A free register cannot be found without liveness. A move also makes the
// code longer, so a function which asked for small code keeps its literals.
@@ -124,6 +164,8 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
if (IsVOPD3 && !ST.hasVOPD3())
return false;
+ if (IsVOPD3 && isVOPD3F64OPYSrcHazard(TII, MIX, MIY))
+ return false;
if (!IsVOPD3 && ((TII.isVOP3(MIX) && !canMapVOP3PToVOPD(MIX)) ||
(TII.isVOP3(MIY) && !canMapVOP3PToVOPD(MIY))))
return false;
diff --git a/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
new file mode 100644
index 0000000000000..e201d813108ca
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
@@ -0,0 +1,355 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=-vopd3-f64-opy-src-hazard -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=NOHAZARD %s
+
+# In a VOPD3 whose OPX is a 64-bit operation an OPY VGPR source reads back the
+# wrong value when it names the last VGPR the wave owns. A wave always owns a
+# whole number of allocation granules, 16 VGPRs each on gfx1250, so the last
+# VGPR it owns is always v(16n-1).
+
+---
+# v15 is a granule boundary, so it may be the last VGPR the wave owns. It is
+# read by src0 of OPY, so the pair must not be formed.
+name: fma_f64_opy_src0_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opy_src0_at_granule_boundary
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opy_src0_at_granule_boundary
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# Same, but read by src1 of OPY.
+name: fma_f64_opy_src1_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opy_src1_at_granule_boundary
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr6 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr0_vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr8, 0, $vgpr15, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opy_src1_at_granule_boundary
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr6 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr0_vgpr1, 0, $vgpr8, 0, $vgpr15, 0, $vgpr6, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr6 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr0_vgpr1, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr8, 0, $vgpr15, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# Same, but read by src2 of OPY.
+name: fma_f64_opy_src2_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opy_src2_at_granule_boundary
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr6 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr8, 0, $vgpr15, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opy_src2_at_granule_boundary
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr6 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr6, 0, $vgpr8, 0, $vgpr15, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr6 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr8, 0, $vgpr15, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# v14 can never be the last VGPR the wave owns, so the pair is fine.
+name: fma_f64_opy_not_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opy_not_at_granule_boundary
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr14 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr14, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opy_not_at_granule_boundary
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr14 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr14, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr14 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr14, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# The rule follows the granule, not a fixed register number: v31 is rejected
+# too.
+name: fma_f64_opy_at_second_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opy_at_second_granule_boundary
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr31 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr31, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opy_at_second_granule_boundary
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr31 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr31, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr31 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr31, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# The call names v16, but a register named only by a call does not contribute
+# to the function's VGPR usage: its own count stays at 16, so v15 can still be
+# the last VGPR the wave owns. MachineRegisterInfo does report v16 as used, so
+# a check driven by a VGPR count taken from it would see 17 here and wrongly
+# conclude that v15 is safe.
+name: fma_f64_opy_at_granule_boundary_with_call
+tracksRegLiveness: true
+frameInfo:
+ hasCalls: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opy_at_granule_boundary_with_call
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $sgpr4_sgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit-def dead $vgpr16
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opy_at_granule_boundary_with_call
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $sgpr4_sgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; NOHAZARD-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit-def dead $vgpr16
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $sgpr4_sgpr5 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit-def dead $vgpr16
+...
+
+---
+# Any 64-bit OPX triggers the hazard, not just V_FMA_F64.
+name: add_f64_opy_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: add_f64_opy_at_granule_boundary
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_ADD_F64_pseudo_e32 $vgpr0_vgpr1, $vgpr2_vgpr3, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: add_f64_opy_at_granule_boundary
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_ADD_F64_pseudo_e32_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_ADD_F64_pseudo_e32 $vgpr0_vgpr1, $vgpr2_vgpr3, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# The 64-bit component is the X component whichever way round the pair appears
+# in program order.
+name: fma_f64_second_in_program_order
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_second_in_program_order
+ ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_second_in_program_order
+ ; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# Only OPY sources are affected: OPX may read a granule boundary register.
+name: fma_f64_opx_src_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: fma_f64_opx_src_at_granule_boundary
+ ; GFX1250: $vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr14_vgpr15, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr1, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: fma_f64_opx_src_at_granule_boundary
+ ; NOHAZARD: $vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr14_vgpr15, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr1, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr1 = IMPLICIT_DEF
+ $vgpr2_vgpr3 = IMPLICIT_DEF
+ $vgpr4_vgpr5 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr14_vgpr15 = IMPLICIT_DEF
+ $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr14_vgpr15, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr1, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+# Only a 64-bit OPX is affected: with a 32-bit OPX, OPY may read a granule
+# boundary register.
+name: mul_f32_opy_at_granule_boundary
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: mul_f32_opy_at_granule_boundary
+ ; GFX1250: $vgpr0 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10, $vgpr9 = V_DUAL_MUL_F32_e32_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0, 0, $vgpr1, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ;
+ ; NOHAZARD-LABEL: name: mul_f32_opy_at_granule_boundary
+ ; NOHAZARD: $vgpr0 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr10, $vgpr9 = V_DUAL_MUL_F32_e32_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0, 0, $vgpr1, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ $vgpr0 = IMPLICIT_DEF
+ $vgpr1 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr8 = IMPLICIT_DEF
+ $vgpr15 = IMPLICIT_DEF
+ $vgpr10 = V_MUL_F32_e32 $vgpr0, $vgpr1, implicit $mode, implicit $exec
+ $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+...
>From 1d543535d4972a982427d4edf83bec601fc9dca6 Mon Sep 17 00:00:00 2001
From: Joseph Nash <joseph.nash at amd.com>
Date: Mon, 21 Sep 2026 13:35:42 -0400
Subject: [PATCH 2/3] [AMDGPU] Address review comments on the VOPD3 FP64 OPY
hazard
Allow an OPY source which sits at a granule boundary when this function
uses a VGPR above it. The wave owns at least as many VGPRs as the
function uses, so such a source has a register above it which the wave
owns too and cannot be the last one. The count can only be raised by a
callee, and no pass after register allocation lowers it, so a decrease
is not possible here and comparing against the local count stays
conservative.
This also removes fma_f64_opy_at_granule_boundary_with_call. A register
which appears only as an implicit-def on a call is a return register, so
the callee defines it and its own count covers it. The case could not
show a count decrease and was misleading.
Take the allocation granule from the target parser instead of
IsaInfo, name the target by its triple in the test, and enable the
feature on the individual gfx125x ISA versions rather than on
FeatureISAVersion12_50_Common, which is shared far more widely.
Assisted-by: Claude Code:claude-opus-5
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 13 ++++---
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 26 +++++++------
.../vopd3-f64-opy-last-vgpr-gfx1250.mir | 37 +++++++------------
3 files changed, 37 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 68d9aad22447a..88493fe3c4b34 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -2464,12 +2464,12 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureRealTrue16Insts,
FeatureVMovB64Inst,
FeatureVMulU64Inst,
- FeatureVOPD3F64OPYSrcHazard,
]>;
def FeatureISAVersion12_50_STRICT : FeatureSet<
!listconcat(FeatureISAVersion12_50_Common.Features,
- [FeatureGFX1250_STRICT,
+ [FeatureVOPD3F64OPYSrcHazard,
+ FeatureGFX1250_STRICT,
FeatureAddressableLocalMemorySize327680,
FeatureLDSBankCount64,
FeatureWMMAN16Insts,
@@ -2500,7 +2500,8 @@ def FeatureISAVersion12_50_STRICT : FeatureSet<
def FeatureISAVersion12_50 : FeatureSet<
!listconcat(FeatureISAVersion12_50_Common.Features,
- [FeatureAddressableLocalMemorySize327680,
+ [FeatureVOPD3F64OPYSrcHazard,
+ FeatureAddressableLocalMemorySize327680,
FeatureLDSBankCount64,
FeatureWMMAN16Insts,
FeatureWMMAF4Insts,
@@ -2531,7 +2532,8 @@ def FeatureISAVersion12_50 : FeatureSet<
def FeatureISAVersion12_51 : FeatureSet<
!listconcat(FeatureISAVersion12_50_Common.Features,
- [FeatureAddressableLocalMemorySize327680,
+ [FeatureVOPD3F64OPYSrcHazard,
+ FeatureAddressableLocalMemorySize327680,
FeatureLDSBankCount64,
FeatureWMMAN16Insts,
FeatureWMMAF4Insts,
@@ -2570,7 +2572,8 @@ def FeatureISAVersion12_Generic: FeatureSet<
def FeatureISAVersion12_5_Generic: FeatureSet<
!listconcat(FeatureISAVersion12_50_Common.Features,
- [FeatureAddressableLocalMemorySize327680,
+ [FeatureVOPD3F64OPYSrcHazard,
+ FeatureAddressableLocalMemorySize327680,
FeatureLDSBankCount64,
FeatureBlock16ConversionScaleInsts,
FeatureSetregVGPRMSBFixup,
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index bcde29f9ab05f..0795f88ad4c15 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -28,6 +28,7 @@
#include "llvm/CodeGen/ScheduleDAGMutation.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/MC/MCInst.h"
+#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -105,14 +106,15 @@ static bool canMapVOP3PToVOPD(const MachineInstr &MI) {
getNamedOp(MI, AMDGPU::OpName::src2).getReg();
}
-// In a VOPD3 whose OPX is a 64-bit operation an OPY VGPR source operand reads
-// back the wrong value if it names the last VGPR the wave owns.
-//
-// The number of VGPRs a wave owns is always a whole number of allocation
-// granules, so the last VGPR it owns is always the one just below a granule
-// boundary. A source which is not there cannot be it, whatever the wave's VGPR
-// count turns out to be. That count is deliberately not consulted: it is only
-// known once the whole call graph has been seen, at emission time.
+// In a VOPD3 whose OPX is a 64-bit operation, an OPY VGPR source operand reads
+// back the wrong value if it is the last VGPR the wave owns.
+// A wave always owns a whole number of VGPR allocation granules, so only a
+// register just below a granule boundary can be the last one. The wave also
+// owns at least as many VGPRs as this function uses, so a source which has a
+// register above it in use here cannot be the last one either.
+// The number of VGPRs the wave is actually given is not available until the
+// assembler has seen the whole module, but that can only come out above this
+// function's own usage, so this is conservatively correct.
static bool isVOPD3F64OPYSrcHazard(const SIInstrInfo &TII,
const MachineInstr &MIX,
const MachineInstr &MIY) {
@@ -128,10 +130,11 @@ static bool isVOPD3F64OPYSrcHazard(const SIInstrInfo &TII,
if (TII.getOpSize(MIX, VDstIdx) != 8)
return false;
- // Targets with this hazard do not support dynamic VGPR allocation.
unsigned Granule =
- AMDGPU::IsaInfo::getVGPRAllocGranule(ST, /*DynamicVGPRBlockSize=*/0);
+ AMDGPU::getVGPRAllocGranule(ST.getTargetID().getGPUKind(), ST.isWave32());
const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ unsigned NumVGPRs = TRI->getNumUsedPhysRegs(
+ MF.getRegInfo(), AMDGPU::VGPR_32RegClass, /*IncludeCalls=*/false);
for (AMDGPU::OpName Name :
{AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
const MachineOperand *Src = TII.getNamedOperand(MIY, Name);
@@ -139,7 +142,8 @@ static bool isVOPD3F64OPYSrcHazard(const SIInstrInfo &TII,
if (!Src || !Src->isReg() ||
!AMDGPU::VGPR_32RegClass.contains(Src->getReg()))
continue;
- if ((TRI->getHWRegIndex(Src->getReg()) + 1) % Granule == 0)
+ unsigned Idx = TRI->getHWRegIndex(Src->getReg());
+ if ((Idx + 1) % Granule == 0 && Idx + 1 >= NumVGPRs)
return true;
}
return false;
diff --git a/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
index e201d813108ca..e5a4f02c72575 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
@@ -1,11 +1,11 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX1250 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=-vopd3-f64-opy-src-hazard -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=NOHAZARD %s
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgpu12.50 -mattr=-vopd3-f64-opy-src-hazard -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=NOHAZARD %s
# In a VOPD3 whose OPX is a 64-bit operation an OPY VGPR source reads back the
-# wrong value when it names the last VGPR the wave owns. A wave always owns a
-# whole number of allocation granules, 16 VGPRs each on gfx1250, so the last
-# VGPR it owns is always v(16n-1).
+# wrong value when it is the last VGPR the wave owns. A wave always owns a
+# whole number of allocation granules, 16 VGPRs each on gfx1250, so only
+# v(16n-1) can be the last one, and only when no higher VGPR is in use.
---
# v15 is a granule boundary, so it may be the last VGPR the wave owns. It is
@@ -179,49 +179,40 @@ body: |
...
---
-# The call names v16, but a register named only by a call does not contribute
-# to the function's VGPR usage: its own count stays at 16, so v15 can still be
-# the last VGPR the wave owns. MachineRegisterInfo does report v16 as used, so
-# a check driven by a VGPR count taken from it would see 17 here and wrongly
-# conclude that v15 is safe.
-name: fma_f64_opy_at_granule_boundary_with_call
+# v15 sits at a granule boundary, but v20 is in use, so the wave owns at least
+# 21 VGPRs and v15 cannot be the last one it owns. The pair is fine.
+name: fma_f64_opy_at_granule_boundary_not_last_vgpr
tracksRegLiveness: true
-frameInfo:
- hasCalls: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_at_granule_boundary_with_call
+ ; GFX1250-LABEL: name: fma_f64_opy_at_granule_boundary_not_last_vgpr
; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $sgpr4_sgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit-def dead $vgpr16
+ ; GFX1250-NEXT: $vgpr20 = IMPLICIT_DEF
+ ; GFX1250-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
;
- ; NOHAZARD-LABEL: name: fma_f64_opy_at_granule_boundary_with_call
+ ; NOHAZARD-LABEL: name: fma_f64_opy_at_granule_boundary_not_last_vgpr
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
; NOHAZARD-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
; NOHAZARD-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
; NOHAZARD-NEXT: $vgpr7 = IMPLICIT_DEF
; NOHAZARD-NEXT: $vgpr8 = IMPLICIT_DEF
; NOHAZARD-NEXT: $vgpr15 = IMPLICIT_DEF
- ; NOHAZARD-NEXT: $sgpr4_sgpr5 = IMPLICIT_DEF
+ ; NOHAZARD-NEXT: $vgpr20 = IMPLICIT_DEF
; NOHAZARD-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
- ; NOHAZARD-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit-def dead $vgpr16
$vgpr0_vgpr1 = IMPLICIT_DEF
$vgpr2_vgpr3 = IMPLICIT_DEF
$vgpr4_vgpr5 = IMPLICIT_DEF
$vgpr7 = IMPLICIT_DEF
$vgpr8 = IMPLICIT_DEF
$vgpr15 = IMPLICIT_DEF
- $sgpr4_sgpr5 = IMPLICIT_DEF
+ $vgpr20 = IMPLICIT_DEF
$vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
$vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
- dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit-def dead $vgpr16
...
---
>From 8d9f77bbba928f1b57de25af204806116b051d4a Mon Sep 17 00:00:00 2001
From: Joseph Nash <joseph.nash at amd.com>
Date: Mon, 21 Sep 2026 13:52:47 -0400
Subject: [PATCH 3/3] [AMDGPU] Test the VOPD3 FP64 OPY hazard on gfx1251
gfx1251 enables the feature too. It produces output identical to
gfx1250, so both run lines share a check prefix rather than duplicating
every assertion.
Assisted-by: Claude Code:claude-opus-5
---
.../vopd3-f64-opy-last-vgpr-gfx1250.mir | 173 +++++++++---------
1 file changed, 87 insertions(+), 86 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
index e5a4f02c72575..aa3d7b3a2afae 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopd3-f64-opy-last-vgpr-gfx1250.mir
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu12.50 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX125X %s
+# RUN: llc -mtriple=amdgpu12.51 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX125X %s
# RUN: llc -mtriple=amdgpu12.50 -mattr=-vopd3-f64-opy-src-hazard -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=NOHAZARD %s
# In a VOPD3 whose OPX is a 64-bit operation an OPY VGPR source reads back the
@@ -14,15 +15,15 @@ name: fma_f64_opy_src0_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_src0_at_granule_boundary
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opy_src0_at_granule_boundary
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opy_src0_at_granule_boundary
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -48,15 +49,15 @@ name: fma_f64_opy_src1_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_src1_at_granule_boundary
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr6 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr0_vgpr1, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr8, 0, $vgpr15, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opy_src1_at_granule_boundary
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr6 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr0_vgpr1, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr8, 0, $vgpr15, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opy_src1_at_granule_boundary
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -82,15 +83,15 @@ name: fma_f64_opy_src2_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_src2_at_granule_boundary
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr6 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr8, 0, $vgpr15, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opy_src2_at_granule_boundary
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr6 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr8, 0, $vgpr15, 0, 0, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opy_src2_at_granule_boundary
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -116,14 +117,14 @@ name: fma_f64_opy_not_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_not_at_granule_boundary
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr14 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr14, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opy_not_at_granule_boundary
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr14 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr14, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opy_not_at_granule_boundary
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -150,15 +151,15 @@ name: fma_f64_opy_at_second_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_at_second_granule_boundary
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr31 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr31, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opy_at_second_granule_boundary
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr31 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr31, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opy_at_second_granule_boundary
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -185,15 +186,15 @@ name: fma_f64_opy_at_granule_boundary_not_last_vgpr
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opy_at_granule_boundary_not_last_vgpr
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr20 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opy_at_granule_boundary_not_last_vgpr
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr20 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opy_at_granule_boundary_not_last_vgpr
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -221,14 +222,14 @@ name: add_f64_opy_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: add_f64_opy_at_granule_boundary
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_ADD_F64_pseudo_e32 $vgpr0_vgpr1, $vgpr2_vgpr3, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: add_f64_opy_at_granule_boundary
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11 = V_ADD_F64_pseudo_e32 $vgpr0_vgpr1, $vgpr2_vgpr3, implicit $mode, implicit $exec
+ ; GFX125X-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: add_f64_opy_at_granule_boundary
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -253,15 +254,15 @@ name: fma_f64_second_in_program_order
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_second_in_program_order
- ; GFX1250: $vgpr0_vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
- ; GFX1250-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_second_in_program_order
+ ; GFX125X: $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr9 = V_FMA_F32_e64 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, 0, 0, implicit $mode, implicit $exec
+ ; GFX125X-NEXT: $vgpr10_vgpr11 = V_FMA_F64_e64 0, $vgpr0_vgpr1, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_second_in_program_order
; NOHAZARD: $vgpr0_vgpr1 = IMPLICIT_DEF
@@ -287,14 +288,14 @@ name: fma_f64_opx_src_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: fma_f64_opx_src_at_granule_boundary
- ; GFX1250: $vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr14_vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr14_vgpr15, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr1, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: fma_f64_opx_src_at_granule_boundary
+ ; GFX125X: $vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10_vgpr11, $vgpr9 = V_DUAL_FMA_F64_e64_X_FMA_F32_e64_e96_gfx1250 0, $vgpr14_vgpr15, 0, $vgpr2_vgpr3, 0, $vgpr4_vgpr5, 0, $vgpr1, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: fma_f64_opx_src_at_granule_boundary
; NOHAZARD: $vgpr1 = IMPLICIT_DEF
@@ -321,13 +322,13 @@ name: mul_f32_opy_at_granule_boundary
tracksRegLiveness: true
body: |
bb.0:
- ; GFX1250-LABEL: name: mul_f32_opy_at_granule_boundary
- ; GFX1250: $vgpr0 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr1 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr7 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr8 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr15 = IMPLICIT_DEF
- ; GFX1250-NEXT: $vgpr10, $vgpr9 = V_DUAL_MUL_F32_e32_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0, 0, $vgpr1, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX125X-LABEL: name: mul_f32_opy_at_granule_boundary
+ ; GFX125X: $vgpr0 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr1 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr7 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr8 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr15 = IMPLICIT_DEF
+ ; GFX125X-NEXT: $vgpr10, $vgpr9 = V_DUAL_MUL_F32_e32_X_FMA_F32_e64_e96_gfx1250 0, $vgpr0, 0, $vgpr1, 0, $vgpr15, 0, $vgpr8, 0, $vgpr7, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
;
; NOHAZARD-LABEL: name: mul_f32_opy_at_granule_boundary
; NOHAZARD: $vgpr0 = IMPLICIT_DEF
More information about the llvm-commits
mailing list