[llvm] [AMDGPU] RewriteMFMAFormStage: single exit copy for same-block uses (PR #209179)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 06:30:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Romanov Vlad (romanovvlad)
<details>
<summary>Changes</summary>
When multiple VGPR-requiring uses of an MFMA dst are in the
same block as the MFMA, share a single AGPR->VGPR copy placed
before the earliest use, instead of creating one per use.
---
Full diff: https://github.com/llvm/llvm-project/pull/209179.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp (+27-12)
- (added) llvm/test/CodeGen/AMDGPU/mfma-no-multiple-copies.mir (+84)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index a4f854beaeebe..d6189f8875dab 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2810,6 +2810,9 @@ bool RewriteMFMAFormStage::rewrite(
}
DenseSet<MachineOperand *> &DstRegSet = ReplaceMap[DstReg];
+ // One AGPR→VGPR copy per dst register, shared by all same-block uses.
+ Register SameBlockCopyReg;
+ MachineInstr *EarliestSameBlockUse = nullptr;
for (MachineOperand *RU : DstReachingUseCopies) {
MachineBasicBlock *RUBlock = RU->getParent()->getParent();
// Just keep track of the reaching use of this register by block. After we
@@ -2819,22 +2822,34 @@ bool RewriteMFMAFormStage::rewrite(
continue;
}
- // Special case, the use is in the same block as the MFMA. Insert the copy
- // just before the use.
- const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(DstReg);
- const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(DstRC);
- Register NewUseReg = DAG.MRI.createVirtualRegister(VGPRRC);
+ // Lazily create the copy register on first same-block use.
+ if (!SameBlockCopyReg.isValid()) {
+ const TargetRegisterClass *DstRC = DAG.MRI.getRegClass(DstReg);
+ const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(DstRC);
+ SameBlockCopyReg = DAG.MRI.createVirtualRegister(VGPRRC);
+ }
+
+ // Track the earliest use for copy insertion point.
MachineInstr *UseInst = RU->getParent();
+ if (!EarliestSameBlockUse ||
+ SlotIndex::isEarlierInstr(
+ DAG.LIS->getInstructionIndex(*UseInst),
+ DAG.LIS->getInstructionIndex(*EarliestSameBlockUse)))
+ EarliestSameBlockUse = UseInst;
+
+ RU->setReg(SameBlockCopyReg);
+ }
+
+ // Insert the copy before the earliest same-block use.
+ if (SameBlockCopyReg.isValid()) {
MachineInstrBuilder VGPRCopy =
- BuildMI(*UseInst->getParent(), UseInst->getIterator(),
- UseInst->getDebugLoc(), TII->get(TargetOpcode::COPY))
- .addDef(NewUseReg, {}, 0)
+ BuildMI(*EarliestSameBlockUse->getParent(),
+ EarliestSameBlockUse->getIterator(),
+ EarliestSameBlockUse->getDebugLoc(),
+ TII->get(TargetOpcode::COPY))
+ .addDef(SameBlockCopyReg, {}, 0)
.addUse(DstReg, {}, 0);
DAG.LIS->InsertMachineInstrInMaps(*VGPRCopy);
- // Since we know this use has only one reaching def, we can replace the
- // use reg.
- RU->setReg(NewUseReg);
- // Track the copy source operand for r eplacement.
DstRegSet.insert(&VGPRCopy->getOperand(1));
}
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-multiple-copies.mir b/llvm/test/CodeGen/AMDGPU/mfma-no-multiple-copies.mir
new file mode 100644
index 0000000000000..98ad1633cbfc5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-multiple-copies.mir
@@ -0,0 +1,84 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=machine-scheduler -amdgpu-disable-rewrite-mfma-form-sched-stage=false %s -o - | FileCheck %s
+
+# Test that only one copy generated for all users of mfma in the same block.
+
+--- |
+ define void @mfma_no_multiple_copies() #0 { ret void }
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+
+---
+name: mfma_no_multiple_copies
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ ; CHECK-LABEL: name: mfma_no_multiple_copies
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p0:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %m3:vreg_128_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY %m3
+ ; CHECK-NEXT: %p1:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p2:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p3:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p4:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p5:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p6:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: %p7:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %m0:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %m1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %m2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit undef $scc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m2, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_128_align2 = COPY [[COPY]]
+ ; CHECK-NEXT: %out1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: %out2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s1:vreg_128_align2, undef %s0:vreg_128_align2, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit %out1, implicit %out2
+ bb.0:
+ successors: %bb.1(0x80000000)
+ %p0:vreg_1024 = IMPLICIT_DEF
+ %p1:vreg_1024 = IMPLICIT_DEF
+ %p2:vreg_1024 = IMPLICIT_DEF
+ %p3:vreg_1024 = IMPLICIT_DEF
+ %p4:vreg_1024 = IMPLICIT_DEF
+ %p5:vreg_1024 = IMPLICIT_DEF
+ %p6:vreg_1024 = IMPLICIT_DEF
+ %p7:vreg_1024 = IMPLICIT_DEF
+
+ %m3:vreg_128_align2 = IMPLICIT_DEF
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+
+ %m0:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m3, 0, 0, 0, implicit $mode, implicit $exec
+ %m1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m0, 0, 0, 0, implicit $mode, implicit $exec
+ %m2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m1, 0, 0, 0, implicit $mode, implicit $exec
+
+ S_CBRANCH_SCC1 %bb.2, implicit undef $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+
+ %m3:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m2, 0, 0, 0, implicit $mode, implicit $exec
+
+ %out1:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s0:vreg_128_align2, undef %s1:vreg_128_align2, %m3, 0, 0, 0, implicit $mode, implicit $exec
+ %out2:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 undef %s1:vreg_128_align2, undef %s0:vreg_128_align2, %m3, 0, 0, 0, implicit $mode, implicit $exec
+
+ KILL %p0, %p1, %p2, %p3, %p4, %p5, %p6, %p7
+
+ S_ENDPGM 0, implicit %out1, implicit %out2
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/209179
More information about the llvm-commits
mailing list