[llvm] [AMDGPU] Fold inlinable constants with V_FMAMK into VOP3 V_FMA (PR #230452)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 9 04:01:22 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Carl Ritson (perlfu)
<details>
<summary>Changes</summary>
V_FMAMK (typically formed during peephole-opt) doesn't support inlinable constants for Src2; however, conversion to V_FMA allows folding and removes mov and register use.
---
Full diff: https://github.com/llvm/llvm-project/pull/230452.diff
3 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFoldOperands.cpp (+42)
- (modified) llvm/test/CodeGen/AMDGPU/fma.f16.ll (+90)
- (modified) llvm/test/CodeGen/AMDGPU/si-fold-kimm.mir (+38)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 0d2698d734564..cb8b24de67115 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -920,6 +920,48 @@ bool SIFoldOperandsImpl::tryAddToFoldList(
MI->setDesc(TII->get(Opc));
}
+ // Special case for v_fmamk_{f16, f32} formed by peephole-opt pass.
+ // If Src2 is a literal that can be represented inline then use v_fma.
+ if ((Opc == AMDGPU::V_FMAMK_F32 || Opc == AMDGPU::V_FMAMK_F16 ||
+ Opc == AMDGPU::V_FMAMK_F16_t16 || Opc == AMDGPU::V_FMAMK_F16_fake16) &&
+ OpNo == 3 && OpToFold.isImm()) {
+ unsigned NewOpc = AMDGPU::V_FMA_F32_e64;
+ switch (Opc) {
+ case AMDGPU::V_FMAMK_F16:
+ NewOpc = AMDGPU::V_FMA_F16_gfx9_e64;
+ break;
+ case AMDGPU::V_FMAMK_F16_t16:
+ NewOpc = AMDGPU::V_FMA_F16_gfx9_t16_e64;
+ break;
+ case AMDGPU::V_FMAMK_F16_fake16:
+ NewOpc = AMDGPU::V_FMA_F16_gfx9_fake16_e64;
+ break;
+ default:
+ break;
+ }
+ MI->setDesc(TII->get(NewOpc));
+ // Add modifiers to src operands
+ for (unsigned OpI = 3; OpI >= 1; --OpI) {
+ MachineOperand *Op = &MI->getOperand(OpI);
+ MI->insert(Op, {MachineOperand::CreateImm(0)});
+ }
+ // Add clamp, omod, op_sel
+ const AMDGPU::OpName Mods[] = {
+ AMDGPU::OpName::clamp, AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
+ for (AMDGPU::OpName Name : Mods) {
+ if (AMDGPU::hasNamedOperand(NewOpc, Name))
+ MI->addOperand(MachineOperand::CreateImm(0));
+ }
+ unsigned NewOpNo =
+ AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src2);
+ bool FoldAsFMA = tryAddToFoldList(FoldList, MI, NewOpNo, OpToFold);
+ if (FoldAsFMA)
+ return true;
+ // Reset back to fmamk
+ TII->removeModOperands(*MI);
+ MI->setDesc(TII->get(Opc));
+ }
+
// Special case for s_fmac_f32 if we are trying to fold into Src2.
// By transforming into fmaak we can untie Src2 and make folding legal.
if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index 365c938d8f6e6..d9dd11a295748 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -785,3 +785,93 @@ bb:
%i5 = zext <2 x i1> %i4 to <2 x i32>
ret <2 x i32> %i5
}
+
+define half @test_imm_fold_to_fma(half %arg) {
+; GFX9-SDAG-LABEL: test_imm_fold_to_fma:
+; GFX9-SDAG: ; %bb.0: ; %bb
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x4900
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, 0.5
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_imm_fold_to_fma:
+; GFX9-GISEL: ; %bb.0: ; %bb
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x4900
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, 0.5
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: test_imm_fold_to_fma:
+; GFX10: ; %bb.0: ; %bb
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_fma_f16 v0, v0, 0x4900, 0.5
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: test_imm_fold_to_fma:
+; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v0.l, 0x4900, 0.5
+; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-FAKE16-LABEL: test_imm_fold_to_fma:
+; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, 0x4900, 0.5
+; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-TRUE16-LABEL: test_imm_fold_to_fma:
+; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_fma_f16 v0.l, v0.l, 0x4900, 0.5
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: test_imm_fold_to_fma:
+; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, 0x4900, 0.5
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-TRUE16-LABEL: test_imm_fold_to_fma:
+; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v0.l, 0x4900, 0.5
+; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-FAKE16-LABEL: test_imm_fold_to_fma:
+; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, 0x4900, 0.5
+; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-TRUE16-LABEL: test_imm_fold_to_fma:
+; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: v_fma_f16 v0.l, v0.l, 0x4900, 0.5
+; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-FAKE16-LABEL: test_imm_fold_to_fma:
+; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, 0x4900, 0.5
+; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+bb:
+ %mul = fmul contract half %arg, 10.0
+ %madmk = fadd contract half %mul, 0.5
+ ret half %madmk
+}
diff --git a/llvm/test/CodeGen/AMDGPU/si-fold-kimm.mir b/llvm/test/CodeGen/AMDGPU/si-fold-kimm.mir
index d22a12bef4ef7..9df8f8e82e3a0 100644
--- a/llvm/test/CodeGen/AMDGPU/si-fold-kimm.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-fold-kimm.mir
@@ -44,6 +44,25 @@ body: |
$vgpr0 = COPY %2
...
+---
+name: fmamk_f16_inlinable_src2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; GCN-LABEL: name: fmamk_f16_inlinable_src2
+ ; GCN: liveins: $vgpr0
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GCN-NEXT: [[V_FMA_F16_gfx9_e64_:%[0-9]+]]:vgpr_32 = contract nofpexcept V_FMA_F16_gfx9_e64 0, [[COPY]], 0, 11878, 0, 14336, 0, 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: $vgpr0 = COPY [[V_FMA_F16_gfx9_e64_]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_MOV_B32_e32 14336, implicit $exec
+ %2:vgpr_32 = contract nofpexcept V_FMAMK_F16 %0, 11878, %1, implicit $mode, implicit $exec
+ $vgpr0 = COPY %2
+...
+
---
name: fmaak_f32
tracksRegLiveness: true
@@ -83,3 +102,22 @@ body: |
%2:vgpr_32 = contract nofpexcept V_FMAMK_F32 %0, 778462824, %1, implicit $mode, implicit $exec
$vgpr0 = COPY %2
...
+
+---
+name: fmamk_f32_inlinable_src2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; GCN-LABEL: name: fmamk_f32_inlinable_src2
+ ; GCN: liveins: $vgpr0
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GCN-NEXT: [[V_FMA_F32_e64_:%[0-9]+]]:vgpr_32 = contract nofpexcept V_FMA_F32_e64 0, [[COPY]], 0, 778462824, 0, 1056964608, 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: $vgpr0 = COPY [[V_FMA_F32_e64_]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_MOV_B32_e32 1056964608, implicit $exec
+ %2:vgpr_32 = contract nofpexcept V_FMAMK_F32 %0, 778462824, %1, implicit $mode, implicit $exec
+ $vgpr0 = COPY %2
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/230452
More information about the llvm-commits
mailing list