[llvm] port eliminate-save-restore from amdgcnas.py to LLVM (PR #190294)
Zeng Wu via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 22:20:54 PDT 2026
https://github.com/zwu-2025 updated https://github.com/llvm/llvm-project/pull/190294
>From cec1e2009bac77451dd1da04e54d28619e0a9a81 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Thu, 2 Apr 2026 21:55:03 +0000
Subject: [PATCH 1/3] PreRA EliminateSaveRestore
---
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 239 ++++++++++++++++--
1 file changed, 212 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index cd56887fd46a8..36cf078c7cdf2 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -36,7 +36,9 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/InitializePasses.h"
using namespace llvm;
@@ -57,6 +59,7 @@ class GCNPreRAOptimizationsImpl {
public:
GCNPreRAOptimizationsImpl(LiveIntervals *LS) : LIS(LS) {}
bool run(MachineFunction &MF);
+ bool eliminateSaveRestore(MachineBasicBlock &);
};
class GCNPreRAOptimizationsLegacy : public MachineFunctionPass {
@@ -238,6 +241,182 @@ GCNPreRAOptimizationsPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
}
+// Eliminate save-modify-restore patterns:
+// v_mov tmp, orig ; save
+// <op> orig, tmp, ... ; modify (overwrites orig, reads tmp)
+// ... ; uses of orig
+// v_mov orig, tmp ; restore
+//
+// Transform to:
+// <op> tmp, orig, ... ; write to tmp instead
+// ... ; replace uses of orig with tmp
+// ; save and restore movs are dead
+bool GCNPreRAOptimizationsImpl::eliminateSaveRestore(MachineBasicBlock &MBB) {
+ if (MBB.empty())
+ return false;
+
+ bool Changed = true;
+ using InstrIt = MachineBasicBlock::iterator;
+ auto IsVMov = [] (InstrIt I) {
+ return I->getOpcode() == AMDGPU::V_MOV_B32_e32 || I->getOpcode() == AMDGPU::V_MOV_B32_e64;
+ };
+
+ auto FindModifyInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp) {
+ InstrIt Next = Start;
+ while (Next != End) {
+ // Mask is changed in the current block and bail out for now.
+ if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+ return End;
+ }
+
+ // def/use are tied which make swap complex, skip for now.
+ for (size_t Idx = 0; Idx < Next->getNumDefs(); Idx++)
+ if (Next->isRegTiedToUseOperand(Idx)) return End;
+
+ for (size_t Idx = Next->getNumDefs(); Idx < Next->getNumOperands(); Idx++)
+ if (Next->isRegTiedToDefOperand(Idx)) return End;
+
+ // If the value of Tmp is changed, bail out.
+ if (Next->modifiesRegister(Tmp, TRI)) return End;
+
+ if (Next->definesRegister(Orig, TRI)) {
+ // If the instru changes Orig and reads Tmp, then it is found.
+ if (Next->readsVirtualRegister(Tmp) && !Next->killsRegister(Tmp, TRI)) break;
+ // Or else the Orig is changed, bail out.
+ return End;
+ }
+
+ ++Next;
+ }
+
+ return Next;
+ };
+
+ auto FindRestoreInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp, llvm::SmallVector<InstrIt, 4> &Candidates) {
+ InstrIt Next = Start;
+ while (Next != End) {
+ // Mask is changed in the current block and bail out for now.
+ if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+ return End;
+ }
+
+ // The instructions in candidates ONLY read Orig, NOT write Orig.
+ // If so, there is an instr which writes to Orig before the v_mov orig, tmp
+ // there will be no v_mov orig, tmp in the future because the MIR is pre-RA SSA form.
+ if (Next->killsRegister(Orig, TRI)) {
+ if (IsVMov(Next) &&
+ Next->getOperand(0).getReg() == Orig &&
+ Next->getOperand(1).isReg() && Next->getOperand(1).getReg() == Tmp) {
+ break;
+ }
+
+ return End;
+ }
+
+ if (Next->killsRegister(Tmp, TRI)) {
+ // Tmp should not be changed so the restore op v_mov orig, tmp is valid.
+ return End;
+ }
+
+ if (Next->readsVirtualRegister(Orig)) Candidates.push_back(Next);
+
+ ++Next;
+ }
+
+ return Next;
+ };
+
+ bool Updated = true;
+ while (Updated) {
+ Updated = false;
+ InstrIt I = MBB.begin();
+ InstrIt E = MBB.end();
+ while (I != E) {
+ // find the start V_MOV tmp, orig
+ if (!IsVMov(I) || !I->getOperand(1).isReg()) {
+ ++I;
+ continue;
+ }
+
+ Register Orig = I->getOperand(1).getReg();
+ Register Tmp = I->getOperand(0).getReg();
+
+ // Skip if the operands are physical reigsters.
+ if (!Tmp.isVirtual() || !Orig.isVirtual()) {
+ ++I;
+ continue;
+ }
+
+ InstrIt Start = I;
+ InstrIt ModifyInst = FindModifyInstr(std::next(Start), E, Orig, Tmp);
+ if (ModifyInst == E) {
+ // There is no instrution which write to orig and read tmp for V_MOV tmp, orig.
+ // Try again from the next V_MOV.
+ I = std::next(Start);
+ continue;
+ }
+
+ // Find the restore op and collect all instructions read-only Orig and no-write to Tmp between it and the NewCopy
+ llvm::SmallVector<InstrIt, 4> Candidates;
+ InstrIt RestoreInstr = FindRestoreInstr(std::next(ModifyInst), E, Orig, Tmp, Candidates);
+ if (RestoreInstr == E) {
+ // there is no restore op V_MOV orig, tmp
+ // try again from the next V_MOV.
+ I = std::next(Start);
+ continue;
+ }
+
+ // swap the operand for orig and tmp
+ for (unsigned Idx = 0; Idx < ModifyInst->getNumOperands(); ++Idx) {
+ MachineOperand &MO = ModifyInst->getOperand(Idx);
+ if (!MO.isReg()) continue;
+ if (MO.isDef() && MO.getReg() == Orig) MO.setReg(Tmp);
+ if (MO.isUse() && MO.getReg() == Tmp) MO.setReg(Orig);
+ }
+
+ // replace Operand for reg Orig with reg Tmp in every instruction in Candidates
+ for (auto &Candidate : Candidates) {
+ MachineInstr *MI = &*Candidate;
+ for (MachineOperand &MO : MI->operands()) {
+ if (MO.isReg() && MO.isUse() && MO.getReg() == Orig) {
+ MO.setReg(Tmp);
+ }
+ }
+ }
+
+ // mark instrution Start and RestoreInstr as dead
+ I = std::next(RestoreInstr);
+ Start->eraseFromParent();
+ RestoreInstr->eraseFromParent();
+ Changed = true;
+
+ // update the liverange or other register info in the current block.
+ LIS->RemoveMachineInstrFromMaps(*Start);
+ LIS->RemoveMachineInstrFromMaps(*RestoreInstr);
+
+ Start->eraseFromParent();
+ RestoreInstr->eraseFromParent();
+
+ MRI->clearKillFlags(Orig);
+ MRI->clearKillFlags(Tmp);
+
+ if (LIS->hasInterval(Orig)) {
+ LIS->removeInterval(Orig);
+ LIS->createAndComputeVirtRegInterval(Orig);
+ }
+
+ if (LIS->hasInterval(Tmp)) {
+ LIS->removeInterval(Tmp);
+ LIS->createAndComputeVirtRegInterval(Tmp);
+ }
+
+ Updated = true;
+ }
+ }
+
+ return Changed;
+}
+
bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
@@ -258,34 +437,40 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
Changed |= processReg(Reg);
}
- if (!ST.useRealTrue16Insts())
- return Changed;
-
- // Add RA hints to improve True16 COPY elimination.
- for (const MachineBasicBlock &MBB : MF) {
- for (const MachineInstr &MI : MBB) {
- if (MI.getOpcode() != AMDGPU::COPY)
- continue;
- Register Dst = MI.getOperand(0).getReg();
- Register Src = MI.getOperand(1).getReg();
- const TargetRegisterClass *DstRC = TRI->getRegClassForReg(*MRI, Dst);
- bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(DstRC);
- if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
- TRI->getRegClassForReg(*MRI, Src) == &AMDGPU::VGPR_32RegClass)
- MRI->setRegAllocationHint(Dst, 0, TRI->getSubReg(Src, AMDGPU::lo16));
- if (Src.isVirtual() &&
- MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass &&
- Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
- MRI->setRegAllocationHint(Src, 0, TRI->getSubReg(Dst, AMDGPU::lo16));
- if (!Dst.isVirtual() || !Src.isVirtual())
- continue;
- if (MRI->getRegClass(Dst) == &AMDGPU::VGPR_32RegClass &&
- MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass) {
- MRI->setRegAllocationHint(Dst, AMDGPURI::Size32, Src);
- MRI->setRegAllocationHint(Src, AMDGPURI::Size16, Dst);
+ if (ST.useRealTrue16Insts()) {
+ // Add RA hints to improve True16 COPY elimination.
+ for (const MachineBasicBlock &MBB : MF) {
+ for (const MachineInstr &MI : MBB) {
+ if (MI.getOpcode() != AMDGPU::COPY)
+ continue;
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ const TargetRegisterClass *DstRC = TRI->getRegClassForReg(*MRI, Dst);
+ bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(DstRC);
+ if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
+ TRI->getRegClassForReg(*MRI, Src) == &AMDGPU::VGPR_32RegClass)
+ MRI->setRegAllocationHint(Dst, 0, TRI->getSubReg(Src, AMDGPU::lo16));
+ if (Src.isVirtual() &&
+ MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass &&
+ Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
+ MRI->setRegAllocationHint(Src, 0, TRI->getSubReg(Dst, AMDGPU::lo16));
+ if (!Dst.isVirtual() || !Src.isVirtual())
+ continue;
+ if (MRI->getRegClass(Dst) == &AMDGPU::VGPR_32RegClass &&
+ MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass) {
+ MRI->setRegAllocationHint(Dst, AMDGPURI::Size32, Src);
+ MRI->setRegAllocationHint(Src, AMDGPURI::Size16, Dst);
+ }
+ if (IsDst16Bit && MRI->getRegClass(Src) == &AMDGPU::VGPR_32RegClass)
+ MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
+ }
}
- if (IsDst16Bit && MRI->getRegClass(Src) == &AMDGPU::VGPR_32RegClass)
- MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
+
+ }
+
+ if (ST.hasGFX950Insts()) {
+ for (MachineBasicBlock &MBB : MF) {
+ Changed |= eliminateSaveRestore(MBB);
}
}
>From 37c6077fd63949ffad658a6ac6cfb4e8d70ee188 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Fri, 3 Apr 2026 00:56:58 +0000
Subject: [PATCH 2/3] PostRA EliminateSaveRestore
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 167 ++++++++++++++++++-
1 file changed, 165 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 411c14fef63b6..c9be7bb6ec115 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -26,6 +26,7 @@
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/Support/BranchProbability.h"
using namespace llvm;
@@ -81,6 +82,8 @@ class SIPreEmitPeephole {
void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
bool IsHiBits, const MachineOperand &SrcMO);
+ bool eliminateSaveRestore(MachineBasicBlock &MBB);
+
public:
bool run(MachineFunction &MF, MachineLoopInfo *MLI);
};
@@ -726,6 +729,160 @@ void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
I.eraseFromParent();
}
+// Before the change:
+// tmp = orig ; SaveInstr
+// orig = OP(tmp, ...) ; ModifyInstr
+// uses(orig) ;
+// orig = tmp ; RestoreInstr
+//
+// After the change:
+// tmp = OP(orig)
+// use(tmp)
+//
+// So, the live range of tmp must be ended here, or else bail out.
+bool SIPreEmitPeephole::eliminateSaveRestore(MachineBasicBlock &MBB) {
+ if (MBB.empty())
+ return false;
+
+ bool Changed = false;
+ const TargetRegisterInfo *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
+ using InstrIt = MachineBasicBlock::iterator;
+
+ // For now, only work on V_MOV_B32 so there will be no physical reigster pair in the IR.
+ auto IsVMov = [](InstrIt I) {
+ return I->getOpcode() == AMDGPU::V_MOV_B32_e32 ||
+ I->getOpcode() == AMDGPU::V_MOV_B32_e64;
+ };
+
+ InstrIt I = MBB.begin();
+ InstrIt E = MBB.end();
+ auto FindModifyInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp) {
+ InstrIt Next = Start;
+ while (Next != End) {
+ // Mask is changed in the current block and bail out for now.
+ if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+ return End;
+ }
+
+ // Def/use are tied which make swap complex, skip for now.
+ for (size_t Idx = 0; Idx < Next->getNumDefs(); Idx++)
+ if (Next->isRegTiedToUseOperand(Idx)) return End;
+
+ for (size_t Idx = Next->getNumDefs(); Idx < Next->getNumOperands(); Idx++)
+ if (Next->isRegTiedToDefOperand(Idx)) return End;
+
+ // If the value of Tmp is changed, bail out.
+ if (Next->modifiesRegister(Tmp, TRI)) return End;
+
+ if (Next->definesRegister(Orig, TRI)) {
+ // If the instru changes Orig and reads Tmp, then it is found.
+ if (Next->readsVirtualRegister(Tmp) && !Next->killsRegister(Tmp, TRI)) break;
+ // Or else the Orig is changed, bail out.
+ return End;
+ }
+
+ ++Next;
+ }
+
+ return Next;
+ };
+
+ auto FindRestoreInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp, llvm::SmallVector<InstrIt, 4> &Candidates) {
+ InstrIt Next = Start;
+ while (Next != End) {
+ // Mask is changed in the current block and bail out for now.
+ if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+ return End;
+ }
+
+ if (IsVMov(Next) && Next->getOperand(0).getReg() == Orig &&
+ Next->getOperand(1).isReg() && Next->getOperand(1).getReg() == Tmp) {
+ if (!Next->killsRegister(Tmp, TRI))
+ return End;
+
+ break;
+ }
+
+ // Tmp and Orig should not be changed so that the restore instr `v_mov orig, tmp` is valid.
+ if (Next->killsRegister(Tmp, TRI) or Next->killsRegister(Orig, TRI))
+ return End;
+
+ if (Next->readsRegister(Orig, TRI))
+ Candidates.push_back(Next);
+
+ ++Next;
+ }
+
+ return Next;
+ };
+
+ while (I != E) {
+ // Find the initial V_MOV tmp, orig
+ if (!IsVMov(I) || !I->getOperand(0).isReg() || !I->getOperand(1).isReg()) {
+ ++I;
+ continue;
+ }
+
+ Register Tmp = I->getOperand(0).getReg();
+ Register Orig = I->getOperand(1).getReg();
+
+ InstrIt SaveInstr = I;
+
+ // Find instruction <op> orig, tmp
+ InstrIt ModifyInstr = FindModifyInstr(std::next(SaveInstr), E, Orig, Tmp);
+ if (ModifyInstr == E) {
+ ++I;
+ continue;
+ }
+
+ // Find the restore, collecting candidates
+ llvm::SmallVector<InstrIt, 4> Candidates;
+ InstrIt RestoreInstr = FindRestoreInstr(std::next(ModifyInstr), E, Orig, Tmp, Candidates);
+ if (RestoreInstr == E) {
+ ++I;
+ continue;
+ }
+
+ // Swap Orig and Tmp in ModifyInstr
+ for (MachineOperand &MO : ModifyInstr->operands()) {
+ if (MO.isReg()) {
+ if (TRI->regsOverlap(MO.getReg(), Orig)) {
+ MO.setReg(Tmp);
+ MO.setIsRenamable(false);
+ } else if (TRI->regsOverlap(MO.getReg(), Tmp)) {
+ MO.setReg(Orig);
+ MO.setIsRenamable(false);
+ }
+ }
+ }
+
+ MachineInstr *LastUse = Candidates.empty() ? nullptr : &*Candidates.back();
+
+ for (auto &Candidate : Candidates) {
+ for (MachineOperand &MO : Candidate->operands()) {
+ if (MO.isReg() && MO.isUse() && TRI->regsOverlap(MO.getReg(), Orig)) {
+ MO.setReg(Tmp);
+ MO.setIsRenamable(false);
+
+ // Clear old kill flags on Orig, and only set kill on Tmp for the LAST use
+ MO.setIsKill(false);
+ if (&*Candidate == LastUse) {
+ MO.setIsKill(true); // Manually kill Tmp at the last replaced use
+ }
+ }
+ }
+ }
+
+ I = std::next(RestoreInstr);
+ SaveInstr->eraseFromParent();
+ RestoreInstr->eraseFromParent();
+
+ Changed = true;
+ }
+
+ return Changed;
+}
+
MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
uint32_t UnpackedOpcode,
bool IsHiBits) {
@@ -846,8 +1003,7 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
// side effects.
// Perform the extra MF scans only for supported archs
- if (!ST.hasGFX940Insts())
- return Changed;
+ if (ST.hasGFX940Insts()) {
for (MachineBasicBlock &MBB : MF) {
// Unpack packed instructions overlapped by MFMAs. This allows the
// compiler to co-issue unpacked instructions with MFMA
@@ -866,6 +1022,13 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
performF32Unpacking(*MI);
}
}
+ }
+
+ if (ST.hasGFX950Insts()) {
+ for (MachineBasicBlock &MBB: MF) {
+ Changed |= eliminateSaveRestore(MBB);
+ }
+ }
return Changed;
}
>From d3656fd976c46ab113f0dd9f3fa1eccbc8b2d41f Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Fri, 3 Apr 2026 06:53:01 +0000
Subject: [PATCH 3/3] port optimize_nops and optimize_buffer_load_m0
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 93 ++++++++++++++++++++
1 file changed, 93 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index c9be7bb6ec115..eb50d4f3fe30a 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -21,7 +21,9 @@
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
@@ -83,6 +85,8 @@ class SIPreEmitPeephole {
bool IsHiBits, const MachineOperand &SrcMO);
bool eliminateSaveRestore(MachineBasicBlock &MBB);
+ bool optimizeBufferLoadM0(MachineBasicBlock &MBB);
+ bool optimizeNops(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF, MachineLoopInfo *MLI);
@@ -883,6 +887,87 @@ bool SIPreEmitPeephole::eliminateSaveRestore(MachineBasicBlock &MBB) {
return Changed;
}
+static bool IsVecBufferLoad (const SIInstrInfo *TII, const MachineInstr &MI) {
+ return MI.mayLoad() && (TII->isMTBUF(MI) || TII->isMTBUF(MI));
+};
+
+// pattern 1: s_mov_b32 m0 --> s_nop 0 --> buffer_load --> mfma
+// pattern 2: s_mov_b32 m0 --> buffer_load --> mfma
+// swap buffer_load and mfma, the remove s_nop 0
+bool SIPreEmitPeephole::optimizeBufferLoadM0(MachineBasicBlock &MBB) {
+ if (MBB.empty())
+ return false;
+
+ bool Changed = false;
+ using InstrIt = MachineBasicBlock::iterator;
+ for (InstrIt I = MBB.begin(), E = MBB.end(); I != E; ++I) {
+ if (!TII->isMFMA(I->getOpcode())) continue;
+
+ if (I == MBB.begin()) continue;
+ InstrIt Prev = std::prev(I);
+
+ if (!IsVecBufferLoad(TII, *Prev))
+ continue;
+ InstrIt BufferLoad = Prev;
+
+ if (Prev == MBB.begin()) continue;
+ Prev = std::prev(Prev);
+
+ InstrIt Nop = Prev->getOpcode() == AMDGPU::S_NOP ? Prev : E;
+
+ if (Prev == MBB.begin()) continue;
+ Prev = std::prev(Prev);
+ if (Prev->getOpcode() != AMDGPU::S_MOV_B32)
+ continue;
+
+ Changed = true;
+ MBB.splice(std::next(I), &MBB, BufferLoad);
+ if (Nop != E)
+ Nop->eraseFromParent();
+ }
+
+ return Changed;
+}
+
+// Note: no hazard check here and blindly follow the rules in optimize_nops at amdgcnas.py
+bool SIPreEmitPeephole::optimizeNops(MachineBasicBlock &MBB) {
+ if (MBB.empty())
+ return false;
+
+ bool Changed = false;
+ using InstrIt = MachineBasicBlock::iterator;
+ for (InstrIt I = MBB.begin(), E = MBB.end(); I != E;) {
+ if (I->getOpcode() != AMDGPU::S_NOP) {
+ ++I;
+ continue;
+ }
+
+ InstrIt Next = std::next(I);
+ // nop is the first or last of MBB, then nop is useless.
+ if (I == MBB.begin() || Next == MBB.end()) {
+ Changed = true;
+ I = I->eraseFromParent();
+ continue;
+ }
+
+ // nop is followed by non buffer_load, then nop is useless.
+ if (!IsVecBufferLoad(TII, *Next)) {
+ I = I->eraseFromParent();
+ Changed = true;
+ continue;
+ }
+
+ InstrIt Prev = std::prev(I);
+ if (!Prev->definesRegister(AMDGPU::M0, TRI)) {
+ Changed = true;
+ I = I->eraseFromParent();
+ continue;
+ }
+ }
+
+ return Changed;
+}
+
MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
uint32_t UnpackedOpcode,
bool IsHiBits) {
@@ -1028,6 +1113,14 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
for (MachineBasicBlock &MBB: MF) {
Changed |= eliminateSaveRestore(MBB);
}
+
+ for (MachineBasicBlock &MBB: MF) {
+ Changed |= optimizeNops(MBB);
+ }
+
+ for (MachineBasicBlock &MBB: MF) {
+ Changed |= optimizeBufferLoadM0(MBB);
+ }
}
return Changed;
More information about the llvm-commits
mailing list