[llvm] port eliminate-save-restore from amdgcnas.py to LLVM (PR #190294)

Zeng Wu via llvm-commits llvm-commits at lists.llvm.org
Mon Apr 6 22:20:54 PDT 2026


https://github.com/zwu-2025 updated https://github.com/llvm/llvm-project/pull/190294

>From cec1e2009bac77451dd1da04e54d28619e0a9a81 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Thu, 2 Apr 2026 21:55:03 +0000
Subject: [PATCH 1/3] PreRA EliminateSaveRestore

---
 .../Target/AMDGPU/GCNPreRAOptimizations.cpp   | 239 ++++++++++++++++--
 1 file changed, 212 insertions(+), 27 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index cd56887fd46a8..36cf078c7cdf2 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -36,7 +36,9 @@
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIRegisterInfo.h"
 #include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/InitializePasses.h"
 
 using namespace llvm;
@@ -57,6 +59,7 @@ class GCNPreRAOptimizationsImpl {
 public:
   GCNPreRAOptimizationsImpl(LiveIntervals *LS) : LIS(LS) {}
   bool run(MachineFunction &MF);
+  bool eliminateSaveRestore(MachineBasicBlock &);
 };
 
 class GCNPreRAOptimizationsLegacy : public MachineFunctionPass {
@@ -238,6 +241,182 @@ GCNPreRAOptimizationsPass::run(MachineFunction &MF,
   return PreservedAnalyses::all();
 }
 
+// Eliminate save-modify-restore patterns:
+// v_mov tmp, orig         ; save
+// <op>  orig, tmp, ...    ; modify (overwrites orig, reads tmp)
+// ...                     ; uses of orig
+// v_mov orig, tmp         ; restore
+//
+// Transform to:
+// <op>  tmp, orig, ...    ; write to tmp instead
+// ...                     ; replace uses of orig with tmp
+// ; save and restore movs are dead
+bool GCNPreRAOptimizationsImpl::eliminateSaveRestore(MachineBasicBlock &MBB) {
+  if (MBB.empty())
+    return false;
+
+  bool Changed = true;
+  using InstrIt = MachineBasicBlock::iterator;
+  auto IsVMov = [] (InstrIt I) {
+      return I->getOpcode() == AMDGPU::V_MOV_B32_e32 || I->getOpcode() == AMDGPU::V_MOV_B32_e64;
+  };
+
+  auto FindModifyInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp) {
+      InstrIt Next = Start;
+      while (Next != End) {
+          // Mask is changed in the current block and bail out for now.
+          if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+              return End;
+          }
+
+          // def/use are tied which make swap complex, skip for now.
+          for (size_t Idx = 0; Idx < Next->getNumDefs(); Idx++)
+              if (Next->isRegTiedToUseOperand(Idx)) return End;
+
+          for (size_t Idx = Next->getNumDefs(); Idx < Next->getNumOperands(); Idx++)
+              if (Next->isRegTiedToDefOperand(Idx)) return End;
+
+          // If the value of Tmp is changed, bail out.
+          if (Next->modifiesRegister(Tmp, TRI)) return End;
+
+          if (Next->definesRegister(Orig, TRI)) {
+              // If the instru changes Orig and reads Tmp, then it is found.
+              if (Next->readsVirtualRegister(Tmp) && !Next->killsRegister(Tmp, TRI)) break;
+              // Or else the Orig is changed, bail out.
+              return End;
+          }
+
+          ++Next;
+      }
+
+      return Next;
+  };
+
+  auto FindRestoreInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp, llvm::SmallVector<InstrIt, 4> &Candidates) {
+      InstrIt Next = Start;
+      while (Next != End) {
+          // Mask is changed in the current block and bail out for now.
+          if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+              return End;
+          }
+
+          // The instructions in candidates ONLY read Orig, NOT write Orig.
+          // If so, there is an instr which writes to Orig before the v_mov orig, tmp
+          // there will be no v_mov orig, tmp in the future because the MIR is pre-RA SSA form.
+          if (Next->killsRegister(Orig, TRI)) {
+              if (IsVMov(Next) &&
+                  Next->getOperand(0).getReg() == Orig &&
+                  Next->getOperand(1).isReg() && Next->getOperand(1).getReg() == Tmp) {
+                  break;
+              }
+
+              return End;
+          }
+
+          if (Next->killsRegister(Tmp, TRI)) {
+              // Tmp should not be changed so the restore op v_mov orig, tmp is valid.
+              return End;
+          }
+
+          if (Next->readsVirtualRegister(Orig)) Candidates.push_back(Next);
+
+          ++Next;
+      }
+
+      return Next;
+  };
+
+  bool Updated = true;
+  while (Updated) {
+    Updated = false;
+    InstrIt I = MBB.begin();
+    InstrIt E = MBB.end();
+    while (I != E) {
+        // find the start V_MOV tmp, orig
+        if (!IsVMov(I) || !I->getOperand(1).isReg()) {
+            ++I;
+            continue;
+        }
+
+        Register Orig = I->getOperand(1).getReg();
+        Register Tmp = I->getOperand(0).getReg();
+
+        // Skip if the operands are physical reigsters.
+        if (!Tmp.isVirtual() || !Orig.isVirtual()) {
+            ++I;
+            continue;
+        }
+
+        InstrIt Start = I;
+        InstrIt ModifyInst = FindModifyInstr(std::next(Start), E, Orig, Tmp);
+        if (ModifyInst == E) {
+            // There is no instrution which write to orig and read tmp for V_MOV tmp, orig.
+            // Try again from the next V_MOV.
+            I = std::next(Start);
+            continue;
+        }
+
+        // Find the restore op and collect all instructions read-only Orig and no-write to Tmp between it and the NewCopy
+        llvm::SmallVector<InstrIt, 4> Candidates;
+        InstrIt RestoreInstr = FindRestoreInstr(std::next(ModifyInst), E, Orig, Tmp, Candidates);
+        if (RestoreInstr == E) {
+            // there is no restore op V_MOV orig, tmp
+            // try again from the next V_MOV.
+            I = std::next(Start);
+            continue;
+        }
+
+        // swap the operand for orig and tmp
+        for (unsigned Idx = 0; Idx < ModifyInst->getNumOperands(); ++Idx) {
+            MachineOperand &MO = ModifyInst->getOperand(Idx);
+            if (!MO.isReg()) continue;
+            if (MO.isDef() && MO.getReg() == Orig) MO.setReg(Tmp);
+            if (MO.isUse() && MO.getReg() == Tmp) MO.setReg(Orig);
+        }
+
+        // replace Operand for reg Orig with reg Tmp in every instruction in Candidates
+        for (auto &Candidate : Candidates) {
+            MachineInstr *MI = &*Candidate;
+            for (MachineOperand &MO : MI->operands()) {
+                if (MO.isReg() && MO.isUse() && MO.getReg() == Orig) {
+                    MO.setReg(Tmp);
+                }
+            }
+        }
+
+        // mark instrution Start and RestoreInstr as dead
+        I = std::next(RestoreInstr);
+        Start->eraseFromParent();
+        RestoreInstr->eraseFromParent();
+        Changed = true;
+
+        // update the liverange or other register info in the current block.
+        LIS->RemoveMachineInstrFromMaps(*Start);
+        LIS->RemoveMachineInstrFromMaps(*RestoreInstr);
+
+        Start->eraseFromParent();
+        RestoreInstr->eraseFromParent();
+
+        MRI->clearKillFlags(Orig);
+        MRI->clearKillFlags(Tmp);
+
+        if (LIS->hasInterval(Orig)) {
+            LIS->removeInterval(Orig);
+            LIS->createAndComputeVirtRegInterval(Orig);
+        }
+
+        if (LIS->hasInterval(Tmp)) {
+            LIS->removeInterval(Tmp);
+            LIS->createAndComputeVirtRegInterval(Tmp);
+        }
+
+        Updated = true;
+      }
+  }
+
+  return Changed;
+}
+
 bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
@@ -258,34 +437,40 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
     Changed |= processReg(Reg);
   }
 
-  if (!ST.useRealTrue16Insts())
-    return Changed;
-
-  // Add RA hints to improve True16 COPY elimination.
-  for (const MachineBasicBlock &MBB : MF) {
-    for (const MachineInstr &MI : MBB) {
-      if (MI.getOpcode() != AMDGPU::COPY)
-        continue;
-      Register Dst = MI.getOperand(0).getReg();
-      Register Src = MI.getOperand(1).getReg();
-      const TargetRegisterClass *DstRC = TRI->getRegClassForReg(*MRI, Dst);
-      bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(DstRC);
-      if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
-          TRI->getRegClassForReg(*MRI, Src) == &AMDGPU::VGPR_32RegClass)
-        MRI->setRegAllocationHint(Dst, 0, TRI->getSubReg(Src, AMDGPU::lo16));
-      if (Src.isVirtual() &&
-          MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass &&
-          Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
-        MRI->setRegAllocationHint(Src, 0, TRI->getSubReg(Dst, AMDGPU::lo16));
-      if (!Dst.isVirtual() || !Src.isVirtual())
-        continue;
-      if (MRI->getRegClass(Dst) == &AMDGPU::VGPR_32RegClass &&
-          MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass) {
-        MRI->setRegAllocationHint(Dst, AMDGPURI::Size32, Src);
-        MRI->setRegAllocationHint(Src, AMDGPURI::Size16, Dst);
+  if (ST.useRealTrue16Insts()) {
+      // Add RA hints to improve True16 COPY elimination.
+      for (const MachineBasicBlock &MBB : MF) {
+          for (const MachineInstr &MI : MBB) {
+              if (MI.getOpcode() != AMDGPU::COPY)
+                  continue;
+              Register Dst = MI.getOperand(0).getReg();
+              Register Src = MI.getOperand(1).getReg();
+              const TargetRegisterClass *DstRC = TRI->getRegClassForReg(*MRI, Dst);
+              bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(DstRC);
+              if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
+                  TRI->getRegClassForReg(*MRI, Src) == &AMDGPU::VGPR_32RegClass)
+                  MRI->setRegAllocationHint(Dst, 0, TRI->getSubReg(Src, AMDGPU::lo16));
+              if (Src.isVirtual() &&
+                  MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass &&
+                  Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
+                  MRI->setRegAllocationHint(Src, 0, TRI->getSubReg(Dst, AMDGPU::lo16));
+              if (!Dst.isVirtual() || !Src.isVirtual())
+                  continue;
+              if (MRI->getRegClass(Dst) == &AMDGPU::VGPR_32RegClass &&
+                  MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass) {
+                  MRI->setRegAllocationHint(Dst, AMDGPURI::Size32, Src);
+                  MRI->setRegAllocationHint(Src, AMDGPURI::Size16, Dst);
+              }
+              if (IsDst16Bit && MRI->getRegClass(Src) == &AMDGPU::VGPR_32RegClass)
+                  MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
+          }
       }
-      if (IsDst16Bit && MRI->getRegClass(Src) == &AMDGPU::VGPR_32RegClass)
-        MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
+
+  }
+
+  if (ST.hasGFX950Insts()) {
+    for (MachineBasicBlock &MBB : MF) {
+      Changed |= eliminateSaveRestore(MBB);
     }
   }
 

>From 37c6077fd63949ffad658a6ac6cfb4e8d70ee188 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Fri, 3 Apr 2026 00:56:58 +0000
Subject: [PATCH 2/3] PostRA EliminateSaveRestore

---
 llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 167 ++++++++++++++++++-
 1 file changed, 165 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 411c14fef63b6..c9be7bb6ec115 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/Support/BranchProbability.h"
 using namespace llvm;
@@ -81,6 +82,8 @@ class SIPreEmitPeephole {
   void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
                          bool IsHiBits, const MachineOperand &SrcMO);
 
+  bool eliminateSaveRestore(MachineBasicBlock &MBB);
+
 public:
   bool run(MachineFunction &MF, MachineLoopInfo *MLI);
 };
@@ -726,6 +729,160 @@ void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
   I.eraseFromParent();
 }
 
+// Before the change:
+// tmp = orig          ; SaveInstr
+// orig = OP(tmp, ...) ; ModifyInstr
+// uses(orig)          ;
+// orig = tmp          ; RestoreInstr
+//
+// After the change:
+// tmp = OP(orig)
+// use(tmp)
+//
+// So, the live range of tmp must be ended here, or else bail out.
+bool SIPreEmitPeephole::eliminateSaveRestore(MachineBasicBlock &MBB) {
+  if (MBB.empty())
+    return false;
+
+  bool Changed = false;
+  const TargetRegisterInfo *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
+  using InstrIt = MachineBasicBlock::iterator;
+
+  // For now, only work on V_MOV_B32 so there will be no physical reigster pair in the IR.
+  auto IsVMov = [](InstrIt I) {
+    return I->getOpcode() == AMDGPU::V_MOV_B32_e32 ||
+           I->getOpcode() == AMDGPU::V_MOV_B32_e64;
+  };
+
+  InstrIt I = MBB.begin();
+  InstrIt E = MBB.end();
+  auto FindModifyInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp) {
+      InstrIt Next = Start;
+      while (Next != End) {
+          // Mask is changed in the current block and bail out for now.
+          if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+              return End;
+          }
+
+          // Def/use are tied which make swap complex, skip for now.
+          for (size_t Idx = 0; Idx < Next->getNumDefs(); Idx++)
+              if (Next->isRegTiedToUseOperand(Idx)) return End;
+
+          for (size_t Idx = Next->getNumDefs(); Idx < Next->getNumOperands(); Idx++)
+              if (Next->isRegTiedToDefOperand(Idx)) return End;
+
+          // If the value of Tmp is changed, bail out.
+          if (Next->modifiesRegister(Tmp, TRI)) return End;
+
+          if (Next->definesRegister(Orig, TRI)) {
+              // If the instru changes Orig and reads Tmp, then it is found.
+              if (Next->readsVirtualRegister(Tmp) && !Next->killsRegister(Tmp, TRI)) break;
+              // Or else the Orig is changed, bail out.
+              return End;
+          }
+
+          ++Next;
+      }
+
+      return Next;
+  };
+
+  auto FindRestoreInstr = [&] (InstrIt Start, InstrIt End, Register Orig, Register Tmp, llvm::SmallVector<InstrIt, 4> &Candidates) {
+      InstrIt Next = Start;
+      while (Next != End) {
+          // Mask is changed in the current block and bail out for now.
+          if (Next->modifiesRegister(AMDGPU::EXEC, TRI)) {
+              return End;
+          }
+
+          if (IsVMov(Next) && Next->getOperand(0).getReg() == Orig &&
+              Next->getOperand(1).isReg() && Next->getOperand(1).getReg() == Tmp) {
+              if (!Next->killsRegister(Tmp, TRI))
+                  return End;
+
+              break;
+          }
+
+          // Tmp and Orig should not be changed so that the restore instr `v_mov orig, tmp` is valid.
+          if (Next->killsRegister(Tmp, TRI) or Next->killsRegister(Orig, TRI))
+              return End;
+
+          if (Next->readsRegister(Orig, TRI))
+              Candidates.push_back(Next);
+
+          ++Next;
+      }
+
+      return Next;
+  };
+
+  while (I != E) {
+    // Find the initial V_MOV tmp, orig
+    if (!IsVMov(I) || !I->getOperand(0).isReg() || !I->getOperand(1).isReg()) {
+      ++I;
+      continue;
+    }
+
+    Register Tmp = I->getOperand(0).getReg();
+    Register Orig = I->getOperand(1).getReg();
+
+    InstrIt SaveInstr = I;
+
+    // Find instruction <op> orig, tmp
+    InstrIt ModifyInstr = FindModifyInstr(std::next(SaveInstr), E, Orig, Tmp);
+    if (ModifyInstr == E) {
+      ++I;
+      continue;
+    }
+
+    // Find the restore, collecting candidates
+    llvm::SmallVector<InstrIt, 4> Candidates;
+    InstrIt RestoreInstr = FindRestoreInstr(std::next(ModifyInstr), E, Orig, Tmp, Candidates);
+    if (RestoreInstr == E) {
+      ++I;
+      continue;
+    }
+
+    // Swap Orig and Tmp in ModifyInstr
+    for (MachineOperand &MO : ModifyInstr->operands()) {
+      if (MO.isReg()) {
+        if (TRI->regsOverlap(MO.getReg(), Orig)) {
+          MO.setReg(Tmp);
+          MO.setIsRenamable(false);
+        } else if (TRI->regsOverlap(MO.getReg(), Tmp)) {
+          MO.setReg(Orig);
+          MO.setIsRenamable(false);
+        }
+      }
+    }
+
+    MachineInstr *LastUse = Candidates.empty() ? nullptr : &*Candidates.back();
+
+    for (auto &Candidate : Candidates) {
+      for (MachineOperand &MO : Candidate->operands()) {
+        if (MO.isReg() && MO.isUse() && TRI->regsOverlap(MO.getReg(), Orig)) {
+          MO.setReg(Tmp);
+          MO.setIsRenamable(false);
+
+          // Clear old kill flags on Orig, and only set kill on Tmp for the LAST use
+          MO.setIsKill(false);
+          if (&*Candidate == LastUse) {
+            MO.setIsKill(true); // Manually kill Tmp at the last replaced use
+          }
+        }
+      }
+    }
+
+    I = std::next(RestoreInstr);
+    SaveInstr->eraseFromParent();
+    RestoreInstr->eraseFromParent();
+
+    Changed = true;
+  }
+
+  return Changed;
+}
+
 MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
                                                         uint32_t UnpackedOpcode,
                                                         bool IsHiBits) {
@@ -846,8 +1003,7 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
   // side effects.
 
   // Perform the extra MF scans only for supported archs
-  if (!ST.hasGFX940Insts())
-    return Changed;
+  if (ST.hasGFX940Insts()) {
   for (MachineBasicBlock &MBB : MF) {
     // Unpack packed instructions overlapped by MFMAs. This allows the
     // compiler to co-issue unpacked instructions with MFMA
@@ -866,6 +1022,13 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
       performF32Unpacking(*MI);
     }
   }
+  }
+
+  if (ST.hasGFX950Insts()) {
+    for (MachineBasicBlock &MBB: MF) {
+      Changed |= eliminateSaveRestore(MBB);
+    }
+  }
 
   return Changed;
 }

>From d3656fd976c46ab113f0dd9f3fa1eccbc8b2d41f Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Fri, 3 Apr 2026 06:53:01 +0000
Subject: [PATCH 3/3] port  optimize_nops and optimize_buffer_load_m0

---
 llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 93 ++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index c9be7bb6ec115..eb50d4f3fe30a 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -21,7 +21,9 @@
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
@@ -83,6 +85,8 @@ class SIPreEmitPeephole {
                          bool IsHiBits, const MachineOperand &SrcMO);
 
   bool eliminateSaveRestore(MachineBasicBlock &MBB);
+  bool optimizeBufferLoadM0(MachineBasicBlock &MBB);
+  bool optimizeNops(MachineBasicBlock &MBB);
 
 public:
   bool run(MachineFunction &MF, MachineLoopInfo *MLI);
@@ -883,6 +887,87 @@ bool SIPreEmitPeephole::eliminateSaveRestore(MachineBasicBlock &MBB) {
   return Changed;
 }
 
+static bool IsVecBufferLoad (const SIInstrInfo *TII, const MachineInstr &MI) {
+    return MI.mayLoad() && (TII->isMTBUF(MI) || TII->isMTBUF(MI));
+};
+
+// pattern 1: s_mov_b32 m0 --> s_nop 0 --> buffer_load --> mfma
+// pattern 2: s_mov_b32 m0 --> buffer_load --> mfma
+// swap buffer_load and mfma, the remove s_nop 0
+bool SIPreEmitPeephole::optimizeBufferLoadM0(MachineBasicBlock &MBB) {
+    if (MBB.empty())
+        return false;
+
+    bool Changed = false;
+    using InstrIt = MachineBasicBlock::iterator;
+    for (InstrIt I = MBB.begin(), E = MBB.end(); I != E; ++I) {
+        if (!TII->isMFMA(I->getOpcode())) continue;
+
+        if (I == MBB.begin()) continue;
+        InstrIt Prev = std::prev(I);
+
+        if (!IsVecBufferLoad(TII, *Prev))
+            continue;
+        InstrIt BufferLoad = Prev;
+
+        if (Prev == MBB.begin()) continue;
+        Prev = std::prev(Prev);
+
+        InstrIt Nop = Prev->getOpcode() == AMDGPU::S_NOP ? Prev : E;
+
+        if (Prev == MBB.begin()) continue;
+        Prev = std::prev(Prev);
+        if (Prev->getOpcode() != AMDGPU::S_MOV_B32)
+            continue;
+
+        Changed = true;
+        MBB.splice(std::next(I), &MBB, BufferLoad);
+        if (Nop != E)
+            Nop->eraseFromParent();
+    }
+
+    return Changed;
+}
+
+// Note: no hazard check here and blindly follow the rules in optimize_nops at amdgcnas.py
+bool SIPreEmitPeephole::optimizeNops(MachineBasicBlock &MBB) {
+    if (MBB.empty())
+        return false;
+
+    bool Changed = false;
+    using InstrIt = MachineBasicBlock::iterator;
+    for (InstrIt I = MBB.begin(), E = MBB.end(); I != E;) {
+        if (I->getOpcode() != AMDGPU::S_NOP) {
+            ++I;
+            continue;
+        }
+
+        InstrIt Next = std::next(I);
+        // nop is the first or last of MBB, then nop is useless.
+        if (I == MBB.begin() || Next == MBB.end()) {
+            Changed = true;
+            I = I->eraseFromParent();
+            continue;
+        }
+
+        // nop is followed by non buffer_load, then nop is useless.
+        if (!IsVecBufferLoad(TII, *Next)) {
+            I = I->eraseFromParent();
+            Changed = true;
+            continue;
+        }
+
+        InstrIt Prev = std::prev(I);
+        if (!Prev->definesRegister(AMDGPU::M0, TRI)) {
+            Changed = true;
+            I = I->eraseFromParent();
+            continue;
+        }
+    }
+
+    return Changed;
+}
+
 MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
                                                         uint32_t UnpackedOpcode,
                                                         bool IsHiBits) {
@@ -1028,6 +1113,14 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
     for (MachineBasicBlock &MBB: MF) {
       Changed |= eliminateSaveRestore(MBB);
     }
+
+    for (MachineBasicBlock &MBB: MF) {
+        Changed |= optimizeNops(MBB);
+    }
+
+    for (MachineBasicBlock &MBB: MF) {
+        Changed |= optimizeBufferLoadM0(MBB);
+    }
   }
 
   return Changed;



More information about the llvm-commits mailing list