[llvm-branch-commits] [llvm] [AMDGPU] Form VOPD dot2 pairs with a literal in src1 (PR #230183)
Adel Ejjeh via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 9 07:55:58 PDT 2026
https://github.com/adelejjeh updated https://github.com/llvm/llvm-project/pull/230183
>From 9a547a8a49537418d4dcd2fa29fc9b518bcb98d7 Mon Sep 17 00:00:00 2001
From: Adel Ejjeh <adel.ejjeh at amd.com>
Date: Thu, 8 Oct 2026 11:04:36 -0500
Subject: [PATCH 1/2] [AMDGPU] Form VOPD dot2 pairs with a literal in src1
A V_DOT2 with a register in src0 and a literal in src1 is matched as if
commuted, and commuted when the VOPD pair is built. This recovers the
pairs lost once MachineCSE stopped leaving those literals in src0.
Co-Authored-By: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp | 9 ++
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 73 ++++++---
llvm/lib/Target/AMDGPU/GCNVOPDUtils.h | 3 +
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +-
.../AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll | 9 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll | 47 ++----
.../AMDGPU/vopd-dot2-commute-imm-src1.mir | 149 ++++++++++++++++++
7 files changed, 232 insertions(+), 62 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
index 2ca4fa50f7f291..0179a6422e98b5 100644
--- a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
@@ -197,6 +197,14 @@ class GCNCreateVOPD {
return Selected;
}
+ // Commute the components the match was checked with commuted.
+ void applyCommutes(const SIInstrInfo &TII, VOPDMatchInfo &Match) {
+ for (unsigned I : {0u, 1u}) {
+ if (Match.Commute[I] && !TII.commuteInstruction(*Match.InOrder[I]))
+ llvm_unreachable("failed to commute VOPD component");
+ }
+ }
+
void materializeLiteral(const SIInstrInfo &TII, VOPDCandidate &Candidate) {
if (!Candidate.needsMaterialization())
return;
@@ -326,6 +334,7 @@ class GCNCreateVOPD {
assignMaterializationRegisters(MBB, Candidates);
SmallVector<VOPDCandidate *, 8> Selected = selectCandidates(Candidates);
for (VOPDCandidate *Candidate : Selected) {
+ applyCommutes(*SII, Candidate->Match);
materializeLiteral(*SII, *Candidate);
Changed |= doReplace(SII, Candidate->Match);
}
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index 5241e01387b3d4..ae0881ae8d8198 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -82,7 +82,11 @@ static const MachineOperand &getNamedOp(const MachineInstr &MI,
// Check if MI is a VOP3P instruction with operands that satisfy the constraints
// for mapping it to a VOP2/VOPD opcode: no modifiers, no clamp, src1 and src2
// are registers (src0 can be register or literal), and src2 is same as dst.
-static bool canMapVOP3PToVOPD(const MachineInstr &MI) {
+// An immediate in src1 is accepted if src0 is a register and the operands can
+// be commuted. \p Commute is set if that is needed.
+static bool canMapVOP3PToVOPD(const SIInstrInfo &TII, const MachineInstr &MI,
+ bool &Commute) {
+ Commute = false;
unsigned Opc = MI.getOpcode();
if (Opc != AMDGPU::V_DOT2_F32_F16 && Opc != AMDGPU::V_DOT2_F32_BF16)
return false;
@@ -93,8 +97,14 @@ static bool canMapVOP3PToVOPD(const MachineInstr &MI) {
if (getNamedOp(MI, AMDGPU::OpName::src1_modifiers).getImm() !=
SISrcMods::OP_SEL_1)
return false;
- if (!getNamedOp(MI, AMDGPU::OpName::src1).isReg())
- return false;
+ if (!getNamedOp(MI, AMDGPU::OpName::src1).isReg()) {
+ if (!(getNamedOp(MI, AMDGPU::OpName::src0).isReg() &&
+ getNamedOp(MI, AMDGPU::OpName::src1).isImm() &&
+ TII.isLegalToSwap(MI, getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
+ getNamedOperandIdx(Opc, AMDGPU::OpName::src1))))
+ return false;
+ Commute = true;
+ }
if (getNamedOp(MI, AMDGPU::OpName::src2_modifiers).getImm() !=
SISrcMods::OP_SEL_1)
return false;
@@ -156,11 +166,10 @@ static bool canMaterializeVOPDLiterals(const MachineFunction &MF) {
!MF.getFunction().hasOptSize();
}
-static bool
-checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
- const MachineInstr &MIY, bool IsVOPD3,
- bool AllowSameVGPR,
- SmallVectorImpl<VOPDLiteralFixup> &LiteralFixups) {
+static bool checkVOPDRegConstraints(
+ const SIInstrInfo &TII, const MachineInstr &MIX, const MachineInstr &MIY,
+ bool IsVOPD3, bool AllowSameVGPR,
+ SmallVectorImpl<VOPDLiteralFixup> &LiteralFixups, bool (&CommuteXY)[2]) {
namespace VOPD = AMDGPU::VOPD;
const MachineFunction *MF = MIX.getMF();
@@ -170,8 +179,12 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
return false;
if (IsVOPD3 && isVOPD3F64OPYSrcHazard(TII, MIX, MIY))
return false;
- if (!IsVOPD3 && ((TII.isVOP3(MIX) && !canMapVOP3PToVOPD(MIX)) ||
- (TII.isVOP3(MIY) && !canMapVOP3PToVOPD(MIY))))
+ // Components which only map once src0 and src1 are commuted. The checks
+ // below look at their operands as they will be after the commute.
+ bool Commute[2] = {false, false};
+ if (!IsVOPD3 &&
+ ((TII.isVOP3(MIX) && !canMapVOP3PToVOPD(TII, MIX, Commute[VOPD::X])) ||
+ (TII.isVOP3(MIY) && !canMapVOP3PToVOPD(TII, MIY, Commute[VOPD::Y]))))
return false;
if (TII.isDPP(MIX) || TII.isDPP(MIY))
return false;
@@ -206,8 +219,12 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
for (auto CompIdx : VOPD::COMPONENTS) {
const MachineInstr &MI = (CompIdx == VOPD::X) ? MIX : MIY;
+ const AMDGPU::OpName Src0Name =
+ Commute[CompIdx] ? AMDGPU::OpName::src1 : AMDGPU::OpName::src0;
+ const AMDGPU::OpName Src1Name =
+ Commute[CompIdx] ? AMDGPU::OpName::src0 : AMDGPU::OpName::src1;
- const MachineOperand &Src0 = *TII.getNamedOperand(MI, AMDGPU::OpName::src0);
+ const MachineOperand &Src0 = *TII.getNamedOperand(MI, Src0Name);
if (Src0.isReg()) {
if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, 0, Src0.getReg()))
return false;
@@ -224,7 +241,7 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
return false;
// Only a 32-bit slot is handled, because the caller produces the value
// with a single S_MOV_B32.
- int OpIdx = getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
+ int OpIdx = getNamedOperandIdx(MI.getOpcode(), Src0Name);
if (TII.getOpSize(MI, OpIdx) != 4)
return false;
const TargetRegisterClass *SlotRC =
@@ -251,8 +268,7 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
if (MI.getDesc().hasImplicitUseOfPhysReg(AMDGPU::VCC))
UniqueScalarRegs.insert(AMDGPU::VCC_LO);
- if (const MachineOperand *Src1 =
- TII.getNamedOperand(MI, AMDGPU::OpName::src1)) {
+ if (const MachineOperand *Src1 = TII.getNamedOperand(MI, Src1Name)) {
if (Src1->isReg()) {
if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, 1, Src1->getReg()))
return false;
@@ -310,6 +326,14 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
auto GetVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) {
const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? MIX : MIY;
+ if (Commute[OpcodeIdx]) {
+ int Src0Idx = getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
+ int Src1Idx = getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
+ if ((int)OperandIdx == Src0Idx)
+ OperandIdx = Src1Idx;
+ else if ((int)OperandIdx == Src1Idx)
+ OperandIdx = Src0Idx;
+ }
const MachineOperand &Operand = MI.getOperand(OperandIdx);
if (Operand.isReg() && TRI->isVectorRegister(MRI, Operand.getReg()))
return Operand.getReg();
@@ -330,6 +354,8 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << MIX
<< "\n\tY: " << MIY << "\n");
LiteralFixups.assign(Fixups);
+ CommuteXY[VOPD::X] = Commute[VOPD::X];
+ CommuteXY[VOPD::Y] = Commute[VOPD::Y];
return true;
}
@@ -358,14 +384,19 @@ tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily,
bool AllowSameVGPR = ST.hasGFX12Insts();
// Only a VOPD3 component can need a fixup; a plain one may hold a literal.
- // checkVOPDRegConstraints() only writes this when it succeeds.
+ // checkVOPDRegConstraints() only writes these when it succeeds.
SmallVector<VOPDLiteralFixup, 2> Fixups;
+ bool CommuteXY[2] = {false, false};
if (FirstCanBeVOPD.X && SecondCanBeVOPD.Y) {
if (checkVOPDRegConstraints(TII, FirstMI, SecondMI, IsVOPD3, AllowSameVGPR,
- Fixups))
+ Fixups, CommuteXY))
return VOPDMatchInfo{
- {&FirstMI, &SecondMI}, 0, IsVOPD3, std::move(Fixups)};
+ {&FirstMI, &SecondMI},
+ 0,
+ IsVOPD3,
+ std::move(Fixups),
+ {CommuteXY[AMDGPU::VOPD::X], CommuteXY[AMDGPU::VOPD::Y]}};
}
if (FirstCanBeVOPD.Y && SecondCanBeVOPD.X) {
@@ -376,9 +407,13 @@ tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily,
if (IsAntiDep && !TII.isVOPDAntidependencyAllowed(SecondMI))
return std::nullopt;
if (checkVOPDRegConstraints(TII, SecondMI, FirstMI, IsVOPD3, AllowSameVGPR,
- Fixups))
+ Fixups, CommuteXY))
return VOPDMatchInfo{
- {&FirstMI, &SecondMI}, 1, IsVOPD3, std::move(Fixups)};
+ {&FirstMI, &SecondMI},
+ 1,
+ IsVOPD3,
+ std::move(Fixups),
+ {CommuteXY[AMDGPU::VOPD::Y], CommuteXY[AMDGPU::VOPD::X]}};
}
return std::nullopt;
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h
index dbb4d074bf6e9e..773516fb832bcc 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h
@@ -51,6 +51,9 @@ struct VOPDMatchInfo {
/// can be built. They all have the same 32-bit value, so one register serves
/// the whole pair. Only a VOPD3 pair can need this.
SmallVector<VOPDLiteralFixup, 2> LiteralFixups;
+ /// Entries in \p InOrder which only pair once commuted. The match was checked
+ /// with them commuted, but they are left unchanged until the pair is built.
+ bool Commute[2] = {false, false};
MachineInstr *getMIX() const { return InOrder[XIdx]; }
MachineInstr *getMIY() const { return InOrder[1 - XIdx]; }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index f589d0299442bc..4790f21559e445 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -231,14 +231,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
AMDGPU::OpName Src0OpName, MachineOperand &Src1,
AMDGPU::OpName Src1OpName) const;
- bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
- unsigned toIdx) const;
bool isNonCommutableDPP(const MachineInstr &MI) const;
MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
unsigned OpIdx0,
unsigned OpIdx1) const override;
public:
+ bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
+ unsigned toIdx) const;
enum TargetOperandFlags {
MO_MASK = 0xf,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll
index 1595e6e8a730d4..9c92af928f0182 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll
@@ -1140,8 +1140,7 @@ define float @v_fdot2_f32_bf16_inline_literal_b_x(<2 x bfloat> %a, <2 x bfloat>
;
; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_x:
; GFX11PLUS: ; %bb.0:
-; GFX11PLUS: v_dot2_f32_bf16 v2, v0, 0x40004000, v2
-; GFX11PLUS: v_dot2_f32_bf16 v5, v4, v3, v5
+; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_bf16 v5, v4, v3
; GFX11PLUS: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %e, <2 x bfloat> %d, float %f, i1 false)
@@ -1158,8 +1157,7 @@ define float @v_fdot2_f32_bf16_inline_literal_b_y(<2 x bfloat> %a, <2 x bfloat>
;
; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_y:
; GFX11PLUS: ; %bb.0:
-; GFX11PLUS: v_dot2_f32_bf16 v2, v1, v0, v2
-; GFX11PLUS: v_dot2_f32_bf16 v5, v3, 0x40004000, v5
+; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, v1, v0 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v3
; GFX11PLUS: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %b, <2 x bfloat> %a, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %f, i1 false)
@@ -1176,8 +1174,7 @@ define float @v_fdot2_f32_bf16_inline_literal_b_xy(<2 x bfloat> %a, <2 x bfloat>
;
; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_xy:
; GFX11PLUS: ; %bb.0:
-; GFX11PLUS: v_dot2_f32_bf16 v2, v0, 0x40004000, v2
-; GFX11PLUS: v_dot2_f32_bf16 v5, v3, 0x40004000, v5
+; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v3
; GFX11PLUS: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %f, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll
index f601b5d6ba3d21..8d9730dcb547f8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll
@@ -826,11 +826,6 @@ define float @v_fdot2_dual_sgpr_src0_y(<2 x half> %a, <2 x half> %b, float %c, <
; GFX11: v_dot2acc_f32_f16 v2, v0, v1
; GFX11: v_dot2acc_f32_f16 v5, s0, v3
; GFX11: v_add_f32_e32 v0, v2, v5
-;
-; GFX1170-GFX12-LABEL: v_fdot2_dual_sgpr_src0_y:
-; GFX1170-GFX12: ; %bb.0:
-; GFX1170-GFX12: v_dual_dot2acc_f32_f16 v2, v0, v1 :: v_dual_dot2acc_f32_f16 v5, s0, v3
-; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false)
%r = fadd float %r0, %r1
@@ -1534,16 +1529,10 @@ define float @v_fdot2_inline_literal_b_x(<2 x half> %a, <2 x half> %b, float %c,
; GFX10: v_dot2c_f32_f16 v5, v4, v3
; GFX10: v_add_f32_e32 v0, v2, v5
;
-; GFX11-LABEL: v_fdot2_inline_literal_b_x:
-; GFX11: ; %bb.0:
-; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, v4, v3
-; GFX11: v_add_f32_e32 v0, v2, v5
-;
-; GFX1170-GFX12-LABEL: v_fdot2_inline_literal_b_x:
-; GFX1170-GFX12: ; %bb.0:
-; GFX1170-GFX12: v_dot2_f32_f16 v2, v0, 0x40004000, v2
-; GFX1170-GFX12: v_dot2_f32_f16 v5, v4, v3, v5
-; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5
+; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_x:
+; GFX11PLUS: ; %bb.0:
+; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, v4, v3
+; GFX11PLUS: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2(<2 x half> %e, <2 x half> %d, float %f, i1 false)
%r = fadd float %r0, %r1
@@ -1569,16 +1558,10 @@ define float @v_fdot2_inline_literal_b_y(<2 x half> %a, <2 x half> %b, float %c,
; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v3
; GFX10: v_add_f32_e32 v0, v2, v5
;
-; GFX11-LABEL: v_fdot2_inline_literal_b_y:
-; GFX11: ; %bb.0:
-; GFX11: v_dual_dot2acc_f32_f16 v2, v1, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3
-; GFX11: v_add_f32_e32 v0, v2, v5
-;
-; GFX1170-GFX12-LABEL: v_fdot2_inline_literal_b_y:
-; GFX1170-GFX12: ; %bb.0:
-; GFX1170-GFX12: v_dot2_f32_f16 v2, v1, v0, v2
-; GFX1170-GFX12: v_dot2_f32_f16 v5, v3, 0x40004000, v5
-; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5
+; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_y:
+; GFX11PLUS: ; %bb.0:
+; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, v1, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3
+; GFX11PLUS: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2(<2 x half> %b, <2 x half> %a, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> <half 2.0, half 2.0>, float %f, i1 false)
%r = fadd float %r0, %r1
@@ -1604,16 +1587,10 @@ define float @v_fdot2_inline_literal_b_xy(<2 x half> %a, <2 x half> %b, float %c
; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v3
; GFX10: v_add_f32_e32 v0, v2, v5
;
-; GFX11-LABEL: v_fdot2_inline_literal_b_xy:
-; GFX11: ; %bb.0:
-; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3
-; GFX11: v_add_f32_e32 v0, v2, v5
-;
-; GFX1170-GFX12-LABEL: v_fdot2_inline_literal_b_xy:
-; GFX1170-GFX12: ; %bb.0:
-; GFX1170-GFX12: v_dot2_f32_f16 v2, v0, 0x40004000, v2
-; GFX1170-GFX12: v_dot2_f32_f16 v5, v3, 0x40004000, v5
-; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5
+; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_xy:
+; GFX11PLUS: ; %bb.0:
+; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3
+; GFX11PLUS: v_add_f32_e32 v0, v2, v5
%r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false)
%r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> <half 2.0, half 2.0>, float %f, i1 false)
%r = fadd float %r0, %r1
diff --git a/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir
new file mode 100644
index 00000000000000..e0402c3fbb20b8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir
@@ -0,0 +1,149 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.00 -run-pass=gcn-create-vopd -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX12 %s
+
+# A V_DOT2_F32_* can only be a VOPD component with a register in src1. With an
+# immediate in src1 and a register in src0 the pair is checked as if the dot2
+# were commuted, and it is commuted when the pair is built.
+
+---
+# dot2 is X, the first instruction.
+name: dot2_imm_src1_x
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-LABEL: name: dot2_imm_src1_x
+ ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_ADD_F32_e32_gfx12 1073758208, $vgpr0, $vgpr2, $vgpr4, $vgpr3, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# dot2 is Y, the second instruction.
+name: dot2_imm_src1_y
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-LABEL: name: dot2_imm_src1_y
+ ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr5, $vgpr2 = V_DUAL_ADD_F32_e32_X_DOT2C_F32_F16_e32_gfx12 $vgpr4, $vgpr3, 1073758208, $vgpr0, $vgpr2, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# V_ADD_U32 can only be Y, so the second instruction (dot2) becomes X.
+name: dot2_imm_src1_second_as_x
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-LABEL: name: dot2_imm_src1_second_as_x
+ ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_ADD_U32_e32_gfx12 1073758208, $vgpr0, $vgpr2, $vgpr4, $vgpr3, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr5 = V_ADD_U32_e32 $vgpr4, $vgpr3, implicit $exec
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# Both components have the same immediate in src1, so they share one literal.
+name: dot2_imm_src1_both_same_literal
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5
+ ; GFX12-LABEL: name: dot2_imm_src1_both_same_literal
+ ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_DOT2C_F32_F16_e32_gfx12 1073758208, $vgpr0, $vgpr2, 1073758208, $vgpr3, $vgpr5, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr5 = V_DOT2_F32_F16 8, $vgpr3, 8, 1073758208, 8, $vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# Two different literals cannot be encoded in one VOPD.
+name: dot2_imm_src1_both_different_literals
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5
+ ; GFX12-LABEL: name: dot2_imm_src1_both_different_literals
+ ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX12-NEXT: $vgpr5 = V_DOT2_F32_F16 8, $vgpr3, 8, 1073758209, 8, $vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr5 = V_DOT2_F32_F16 8, $vgpr3, 8, 1073758209, 8, $vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# Immediates in both src0 and src1: commuting would not put a register in src1.
+name: dot2_imm_src0_and_src1
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-LABEL: name: dot2_imm_src0_and_src1
+ ; GFX12: liveins: $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2 = V_DOT2_F32_F16 8, 1073758208, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX12-NEXT: $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr2 = V_DOT2_F32_F16 8, 1073758208, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# Once commuted, src1 of the dot2 is v0, which is in the same bank as src1 of
+# the other component (v8), so the pair is rejected. Without the commute only
+# src0 would be compared (v0 and v1), which do not conflict.
+name: dot2_imm_src1_commuted_bank_conflict
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr8
+ ; GFX12-LABEL: name: dot2_imm_src1_commuted_bank_conflict
+ ; GFX12: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr8
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX12-NEXT: $vgpr5 = V_ADD_F32_e32 $vgpr1, $vgpr8, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr5 = V_ADD_F32_e32 $vgpr1, $vgpr8, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
+
+---
+# Once commuted, src0 of the dot2 is the literal and src1 is v0, which does not
+# conflict with v4 and v6. Without the commute src0 would be v0, which is in the
+# same bank as v4.
+name: dot2_imm_src1_commuted_no_bank_conflict
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2, $vgpr4, $vgpr6
+ ; GFX12-LABEL: name: dot2_imm_src1_commuted_no_bank_conflict
+ ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr4, $vgpr6
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_ADD_F32_e32_gfx12 1073758208, $vgpr0, $vgpr2, $vgpr4, $vgpr6, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+ $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr6, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5
+...
>From d32e4b7364074b4e5aea398c6690e898b12fa431 Mon Sep 17 00:00:00 2001
From: Adel Ejjeh <adel.ejjeh at amd.com>
Date: Thu, 8 Oct 2026 16:00:38 -0500
Subject: [PATCH 2/2] [AMDGPU] Address review comments
Co-Authored-By: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 14 +++++++++-----
.../CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir | 2 +-
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index ae0881ae8d8198..bc3dafc88e53c7 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -97,11 +97,15 @@ static bool canMapVOP3PToVOPD(const SIInstrInfo &TII, const MachineInstr &MI,
if (getNamedOp(MI, AMDGPU::OpName::src1_modifiers).getImm() !=
SISrcMods::OP_SEL_1)
return false;
- if (!getNamedOp(MI, AMDGPU::OpName::src1).isReg()) {
- if (!(getNamedOp(MI, AMDGPU::OpName::src0).isReg() &&
- getNamedOp(MI, AMDGPU::OpName::src1).isImm() &&
- TII.isLegalToSwap(MI, getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
- getNamedOperandIdx(Opc, AMDGPU::OpName::src1))))
+ int Src0Idx = getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
+ int Src1Idx = getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
+ const MachineOperand &Src0 = MI.getOperand(Src0Idx);
+ const MachineOperand &Src1 = MI.getOperand(Src1Idx);
+ // VOPD needs a register in src1. If src1 is an immediate, the dot2 can still
+ // be used once src0 and src1 are swapped.
+ if (!Src1.isReg()) {
+ if (!Src0.isReg() || !Src1.isImm() ||
+ !TII.isLegalToSwap(MI, Src0Idx, Src1Idx))
return false;
Commute = true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir
index e0402c3fbb20b8..d40e564932618b 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu12.00 -run-pass=gcn-create-vopd -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX12 %s
+# RUN: llc -mtriple=amdgpu12.00 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX12 %s
# A V_DOT2_F32_* can only be a VOPD component with a register in src1. With an
# immediate in src1 and a register in src0 the pair is checked as if the dot2
More information about the llvm-branch-commits
mailing list