[llvm] [AMDGPU] Post-RA Peephole for latency-hiding and hazard-avoidance bet… (PR #191319)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 9 16:08:14 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Zeng Wu (zwu-2025)
<details>
<summary>Changes</summary>
Post-RA Peephole for latency-hiding and hazard-avoidance between buffer_load and mfma
In the pass of pre-emit-peephole, change the patterns:
- s_mov_b32 m0 --> s_nop 0 --> buffer_load --> mfma
- s_mov_b32 m0 --> buffer_load --> mfma
into mfma --> buffer_load
by swap buffer_load and mfma and remove s_nop if valid.
---
Full diff: https://github.com/llvm/llvm-project/pull/191319.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp (+92)
- (added) llvm/test/CodeGen/AMDGPU/amdgcn-pre-emit-peephole-buffer-load-mfma.mir (+66)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 411c14fef63b6..a302678d84684 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -21,10 +21,15 @@
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIRegisterInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/Support/BranchProbability.h"
@@ -80,6 +85,7 @@ class SIPreEmitPeephole {
// appropriate source modifers and operands into the unpacked instructions.
void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
bool IsHiBits, const MachineOperand &SrcMO);
+ bool optimizeBufferLoadM0(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF, MachineLoopInfo *MLI);
@@ -726,6 +732,86 @@ void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
I.eraseFromParent();
}
+
+static bool IsVecBufferLoad (const SIInstrInfo *TII, const MachineInstr &MI) {
+ return MI.mayLoad() && (TII->isMTBUF(MI) || TII->isMUBUF(MI));
+};
+
+static MachineInstr *getPrevNonDebugInst(MachineInstr *MI) {
+ for (MachineInstr *I = MI->getPrevNode(); I; I = I->getPrevNode())
+ if (!I->isDebugInstr())
+ return I;
+ return nullptr;
+}
+
+// return true if all of three are true:
+// no flow dependency from BufferLoad to MFMA
+// no anti-flow dependency from BufferLoad to MFMA
+// no output dependency between BufferLoad and MFMA
+static bool canbeReordered(MachineInstr *BufferLoad, MachineInstr *MFMA, const SIRegisterInfo *TRI) {
+ bool HasFlowDep = llvm::any_of(BufferLoad->defs(), [MFMA, TRI](const auto &def) {
+ return def.isReg() && MFMA->readsRegister(def.getReg(), TRI);
+ });
+
+ bool HasAntiDep = llvm::any_of(MFMA->defs(), [BufferLoad, TRI](const auto &def) {
+ return def.isReg() && BufferLoad->readsRegister(def.getReg(), TRI);
+ });
+
+ bool HasOutputDep = llvm::any_of(MFMA->defs(), [BufferLoad, TRI](const auto &def) {
+ return def.isReg() && BufferLoad->modifiesRegister(def.getReg(), TRI);
+ });
+
+ return !HasFlowDep && !HasAntiDep && !HasOutputDep;
+}
+
+// pattern 1: s_mov_b32 m0 --> s_nop 0 --> buffer_load --> mfma
+// pattern 2: s_mov_b32 m0 --> buffer_load --> mfma
+// swap buffer_load and mfma, the remove s_nop 0
+bool SIPreEmitPeephole::optimizeBufferLoadM0(MachineBasicBlock &MBB) {
+ if (MBB.empty())
+ return false;
+
+ bool Changed = false;
+ using InstrIt = MachineBasicBlock::iterator;
+ for (InstrIt I = MBB.begin(), E = MBB.end(); I != E; ++I) {
+ if (!TII->isMFMA(I->getOpcode())) continue;
+ MachineInstr *MFMA = &*I;
+
+ if (I == MBB.begin()) continue;
+ MachineInstr *Prev = getPrevNonDebugInst(&*I);
+ if (Prev == nullptr || !IsVecBufferLoad(TII, *Prev) || !Prev->readsRegister(AMDGPU::M0, TRI))
+ continue;
+
+ MachineInstr *BufferLoad = Prev;
+ Prev = getPrevNonDebugInst(Prev);
+ if (Prev == nullptr) continue;
+
+ MachineInstr *MaybeNop = Prev;
+ MachineInstr *MaybeSMov = nullptr;
+ // In the current lowering pipeline, the post-RA-hazard-rec is just after this pass,
+ // so there is no s_nop. But we still do the check here in case the order of passes is changed.
+ if (MaybeNop->getOpcode() == AMDGPU::S_NOP && MaybeNop->getOperand(0).getImm() == 0) {
+ MaybeSMov = getPrevNonDebugInst(MaybeNop);
+ } else {
+ MaybeSMov = MaybeNop;
+ MaybeNop = nullptr;
+ }
+
+ if (MaybeSMov == nullptr || MaybeSMov->getOpcode() != AMDGPU::S_MOV_B32 ||
+ !MaybeSMov->modifiesRegister(AMDGPU::M0, TRI))
+ continue;
+
+ if (!canbeReordered(BufferLoad, MFMA, TRI)) continue;
+
+ Changed = true;
+ MBB.splice(std::next(I), &MBB, BufferLoad);
+ if (MaybeNop != nullptr)
+ MaybeNop->eraseFromParent();
+ }
+
+ return Changed;
+}
+
MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
uint32_t UnpackedOpcode,
bool IsHiBits) {
@@ -867,5 +953,11 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
}
}
+ if (ST.hasGFX950Insts()) {
+ for (MachineBasicBlock &MBB: MF) {
+ Changed |= optimizeBufferLoadM0(MBB);
+ }
+ }
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-pre-emit-peephole-buffer-load-mfma.mir b/llvm/test/CodeGen/AMDGPU/amdgcn-pre-emit-peephole-buffer-load-mfma.mir
new file mode 100644
index 0000000000000..005418ddac738
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-pre-emit-peephole-buffer-load-mfma.mir
@@ -0,0 +1,66 @@
+# RUN: llc -march=amdgcn -mcpu=gfx950 -run-pass si-pre-emit-peephole -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: test_m0_sched_basic
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21
+
+ ; CHECK-LABEL: bb.0:
+ ; CHECK: S_MOV_B32
+ ; CHECK: V_MFMA_F32
+ ; CHECK: BUFFER_LOAD_DWORD_IDXEN
+
+ $m0 = S_MOV_B32 $sgpr4
+ S_NOP 0
+ $vgpr32 = BUFFER_LOAD_DWORD_IDXEN $vgpr64, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $m0, implicit $exec
+ renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $mode, implicit $exec
+
+
+---
+name: test_m0_sched_basic_no_nop
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21
+
+ ; CHECK-LABEL: bb.0:
+ ; CHECK: S_MOV_B32
+ ; CHECK: V_MFMA_F32
+ ; CHECK: BUFFER_LOAD_DWORD_IDXEN
+
+ $m0 = S_MOV_B32 $sgpr4
+ $vgpr32 = BUFFER_LOAD_DWORD_IDXEN $vgpr64, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $m0, implicit $exec
+ renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $mode, implicit $exec
+
+---
+name: test_output_dependency
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21
+
+ ; CHECK-LABEL: bb.0:
+ ; CHECK: BUFFER_LOAD_DWORD_IDXEN
+ ; CHECK: V_MFMA_F32
+
+ $m0 = S_MOV_B32 $sgpr4
+ $vgpr0 = BUFFER_LOAD_DWORD_IDXEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $m0, implicit $exec
+ renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $mode, implicit $exec
+
+
+---
+name: test_no_m0
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21
+
+ ; CHECK-LABEL: bb.0:
+ ; CHECK: BUFFER_LOAD_DWORD_OFFEN
+ ; CHECK: V_MFMA_F32
+
+ $m0 = S_MOV_B32 $sgpr4
+ $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+ renamable $vgpr0_vgpr1_vgpr2_vgpr3 = nofpexcept V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64 $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $mode, implicit $exec
``````````
</details>
https://github.com/llvm/llvm-project/pull/191319
More information about the llvm-commits
mailing list