[llvm] [MachineLICM] Rematerialize instructions that may be hoisted before LICM (PR #158479)

via llvm-commits llvm-commits at lists.llvm.org
Sun Sep 14 06:37:51 PDT 2025


https://github.com/dianqk updated https://github.com/llvm/llvm-project/pull/158479

>From 8cb8bb00cce43634330626e5224cefb46696919c Mon Sep 17 00:00:00 2001
From: dianqk <dianqk at dianqk.net>
Date: Sat, 13 Sep 2025 21:40:55 +0800
Subject: [PATCH 1/2] [MachineLICM] Rematerialize instructions that may be
 hoisted before LICM

---
 .../llvm/CodeGen/MachineCycleAnalysis.h       |   9 ++
 llvm/lib/CodeGen/MachineCycleAnalysis.cpp     |  45 ++++++
 llvm/lib/CodeGen/MachineLICM.cpp              | 135 +++++++++++++++---
 llvm/lib/CodeGen/MachineSink.cpp              |  67 +--------
 4 files changed, 175 insertions(+), 81 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
index 622f529939bcb..8382df36eed18 100644
--- a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
+++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
@@ -48,6 +48,15 @@ class LLVM_ABI MachineCycleInfoWrapperPass : public MachineFunctionPass {
 //       version.
 LLVM_ABI bool isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I);
 
+/// Returns true if this machine instruction loads from global offset table or
+/// constant pool.
+bool mayLoadFromGOTOrConstantPool(MachineInstr &MI);
+
+/// Returns true if this machine instruction can be a sink candidate.
+bool isSinkIntoCycleCandidate(MachineInstr &MI, MachineCycle *Cycle,
+                              MachineRegisterInfo *MRI,
+                              const TargetInstrInfo *TII);
+
 class MachineCycleAnalysis : public AnalysisInfoMixin<MachineCycleAnalysis> {
   friend AnalysisInfoMixin<MachineCycleAnalysis>;
   LLVM_ABI static AnalysisKey Key;
diff --git a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
index 33a5b664826b3..6ffdb1e59a18d 100644
--- a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
@@ -167,3 +167,48 @@ bool llvm::isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I) {
   // If we got this far, the instruction is cycle invariant!
   return true;
 }
+
+bool llvm::mayLoadFromGOTOrConstantPool(MachineInstr &MI) {
+  assert(MI.mayLoad() && "Expected MI that loads!");
+
+  // If we lost memory operands, conservatively assume that the instruction
+  // reads from everything..
+  if (MI.memoperands_empty())
+    return true;
+
+  for (MachineMemOperand *MemOp : MI.memoperands())
+    if (const PseudoSourceValue *PSV = MemOp->getPseudoValue())
+      if (PSV->isGOT() || PSV->isConstantPool())
+        return true;
+
+  return false;
+}
+
+bool llvm::isSinkIntoCycleCandidate(MachineInstr &MI, MachineCycle *Cycle,
+                                    MachineRegisterInfo *MRI,
+                                    const TargetInstrInfo *TII) {
+  // Not sinking meta instruction.
+  if (MI.isMetaInstruction())
+    return false;
+  // Instruction not a candidate for this target.
+  if (!TII->shouldSink(MI))
+    return false;
+  // Instruction is not cycle invariant.
+  if (!isCycleInvariant(Cycle, MI))
+    return false;
+  // Instruction not safe to move.
+  bool DontMoveAcrossStore = true;
+  if (!MI.isSafeToMove(DontMoveAcrossStore))
+    return false;
+  // Dont sink GOT or constant pool loads.
+  if (MI.mayLoad() && !mayLoadFromGOTOrConstantPool(MI))
+    return false;
+  if (MI.isConvergent())
+    return false;
+  const MachineOperand &MO = MI.getOperand(0);
+  if (!MO.isReg() || !MO.getReg() || !MO.isDef())
+    return false;
+  if (!MRI->hasOneDef(MO.getReg()))
+    return false;
+  return true;
+}
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 4f164e2d53460..804a40612e85d 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -98,6 +98,12 @@ DisableHoistingToHotterBlocks("disable-hoisting-to-hotter-blocks",
                               clEnumValN(UseBFI::All, "all",
                               "enable the feature with/wo profile data")));
 
+static cl::opt<bool> SinkInstsIntoCycleBeforeLICM(
+    "sink-insts-before-licm",
+    cl::desc("Sink instructions into cycles to avoid "
+             "register spills"),
+    cl::init(true), cl::Hidden);
+
 STATISTIC(NumHoisted,
           "Number of machine instructions hoisted out of loops");
 STATISTIC(NumLowRP,
@@ -287,6 +293,8 @@ namespace {
     bool isTgtHotterThanSrc(MachineBasicBlock *SrcBlock,
                             MachineBasicBlock *TgtBlock);
     MachineBasicBlock *getOrCreatePreheader(MachineLoop *CurLoop);
+
+    bool rematerializeIntoCycle(MachineCycle *Cycle, MachineInstr &I);
   };
 
   class MachineLICMBase : public MachineFunctionPass {
@@ -304,7 +312,11 @@ namespace {
         AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
       AU.addRequired<MachineDominatorTreeWrapperPass>();
       AU.addRequired<AAResultsWrapperPass>();
+      if (PreRegAlloc)
+        AU.addRequired<MachineCycleInfoWrapperPass>();
       AU.addPreserved<MachineLoopInfoWrapperPass>();
+      if (PreRegAlloc)
+        AU.addPreserved<MachineCycleInfoWrapperPass>();
       MachineFunctionPass::getAnalysisUsage(AU);
     }
   };
@@ -348,6 +360,7 @@ INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
 INITIALIZE_PASS_END(EarlyMachineLICM, "early-machinelicm",
                     "Early Machine Loop Invariant Code Motion", false, false)
 
@@ -396,6 +409,26 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << MF.getName() << " ********\n");
 
   if (PreRegAlloc) {
+    if (SinkInstsIntoCycleBeforeLICM) {
+      auto *CI = GET_RESULT(MachineCycle, getCycleInfo, Info);
+      SmallVector<MachineCycle *, 8> Cycles(CI->toplevel_cycles());
+      for (auto *Cycle : Cycles) {
+        MachineBasicBlock *Preheader = Cycle->getCyclePreheader();
+        if (!Preheader) {
+          LLVM_DEBUG(dbgs() << "Rematerialization: Can't find preheader\n");
+          continue;
+        }
+        SmallVector<MachineInstr *, 8> Candidates;
+        for (auto &MI : *Preheader)
+          if (isSinkIntoCycleCandidate(MI, Cycle, MRI, TII))
+            Candidates.push_back(&MI);
+        // Walk the candidates in reverse order so that we start with the use
+        // of a def-use chain, if there is any.
+        for (MachineInstr *I : llvm::reverse(Candidates))
+          if (rematerializeIntoCycle(Cycle, *I))
+            Changed = true;
+      }
+    }
     // Estimate register pressure during pre-regalloc pass.
     unsigned NumRPS = TRI->getNumRegPressureSets();
     RegPressure.resize(NumRPS);
@@ -1005,24 +1038,6 @@ MachineLICMImpl::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen,
   return Cost;
 }
 
-/// Return true if this machine instruction loads from global offset table or
-/// constant pool.
-static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI) {
-  assert(MI.mayLoad() && "Expected MI that loads!");
-
-  // If we lost memory operands, conservatively assume that the instruction
-  // reads from everything..
-  if (MI.memoperands_empty())
-    return true;
-
-  for (MachineMemOperand *MemOp : MI.memoperands())
-    if (const PseudoSourceValue *PSV = MemOp->getPseudoValue())
-      if (PSV->isGOT() || PSV->isConstantPool())
-        return true;
-
-  return false;
-}
-
 // This function iterates through all the operands of the input store MI and
 // checks that each register operand statisfies isCallerPreservedPhysReg.
 // This means, the value being stored and the address where it is being stored
@@ -1744,6 +1759,88 @@ bool MachineLICMImpl::isTgtHotterThanSrc(MachineBasicBlock *SrcBlock,
   return Ratio > BlockFrequencyRatioThreshold;
 }
 
+/// Rematerialize instructions into cycles before Machine LICM,
+/// since LICM in the middle-end hoisted every instructions without considering
+/// register pressure.
+bool MachineLICMImpl::rematerializeIntoCycle(MachineCycle *Cycle,
+                                             MachineInstr &I) {
+  LLVM_DEBUG(dbgs() << "Rematerialization: Finding sink block for: " << I);
+  MachineBasicBlock *Preheader = Cycle->getCyclePreheader();
+  assert(Preheader && "Cycle sink needs a preheader block");
+  MachineBasicBlock *SinkBlock = nullptr;
+  const MachineOperand &MO = I.getOperand(0);
+  for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) {
+    LLVM_DEBUG(dbgs() << "Rematerialization:   Analysing use: " << MI);
+    if (!Cycle->contains(MI.getParent())) {
+      LLVM_DEBUG(
+          dbgs() << "Rematerialization:   Use not in cycle, can't sink.\n");
+      return false;
+    }
+    if (!SinkBlock) {
+      SinkBlock = MI.getParent();
+      LLVM_DEBUG(dbgs() << "Rematerialization:   Setting sink block to: "
+                        << printMBBReference(*SinkBlock) << "\n");
+      continue;
+    }
+    if (MI.isPHI()) {
+      for (unsigned I = 1; I != MI.getNumOperands(); I += 2) {
+        Register SrcReg = MI.getOperand(I).getReg();
+        if (TRI->regsOverlap(SrcReg, MO.getReg())) {
+          MachineBasicBlock *SrcBB = MI.getOperand(I + 1).getMBB();
+          if (SrcBB != SinkBlock) {
+            SinkBlock =
+                MDTU->getDomTree().findNearestCommonDominator(SinkBlock, SrcBB);
+            if (!SinkBlock)
+              break;
+          }
+        }
+      }
+    } else {
+      SinkBlock = MDTU->getDomTree().findNearestCommonDominator(SinkBlock,
+                                                                MI.getParent());
+    }
+    if (!SinkBlock) {
+      LLVM_DEBUG(
+          dbgs() << "Rematerialization:   Can't find nearest dominator\n");
+      return false;
+    }
+    LLVM_DEBUG(
+        dbgs() << "Rematerialization:   Setting nearest common dom block: "
+               << printMBBReference(*SinkBlock) << "\n");
+  }
+  if (!SinkBlock) {
+    LLVM_DEBUG(
+        dbgs() << "Rematerialization: Not sinking, can't find sink block.\n");
+    return false;
+  }
+  if (SinkBlock == Preheader) {
+    LLVM_DEBUG(
+        dbgs()
+        << "Rematerialization: Not sinking, sink block is the preheader\n");
+    return false;
+  }
+  for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) {
+    if (MI.isPHI() && MI.getParent() == SinkBlock) {
+      LLVM_DEBUG(dbgs() << "Rematerialization: Not sinking, sink block is "
+                           "using it on PHI.\n");
+      return false;
+    }
+  }
+  LLVM_DEBUG(dbgs() << "Rematerialization: Sinking instruction!\n");
+  SinkBlock->splice(SinkBlock->SkipPHIsAndLabels(SinkBlock->begin()), Preheader,
+                    I);
+  // Conservatively clear any kill flags on uses of sunk instruction
+  for (MachineOperand &MO : I.operands()) {
+    if (MO.isReg() && MO.readsReg())
+      MRI->clearKillFlags(MO.getReg());
+  }
+  // The instruction is moved from its basic block, so do not retain the
+  // debug information.
+  assert(!I.isDebugInstr() && "Should not sink debug inst");
+  I.setDebugLoc(DebugLoc());
+  return true;
+}
+
 template <typename DerivedT, bool PreRegAlloc>
 PreservedAnalyses MachineLICMBasePass<DerivedT, PreRegAlloc>::run(
     MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
@@ -1751,6 +1848,8 @@ PreservedAnalyses MachineLICMBasePass<DerivedT, PreRegAlloc>::run(
   if (!Changed)
     return PreservedAnalyses::all();
   auto PA = getMachineFunctionPassPreservedAnalyses();
+  if (PreRegAlloc)
+    PA.preserve<MachineCycleAnalysis>();
   PA.preserve<MachineLoopAnalysis>();
   return PA;
 }
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 9ec5151a039b7..615edf6b414da 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -39,6 +39,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineLoopUtils.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -258,9 +259,6 @@ class MachineSinking {
                                       bool &BreakPHIEdge,
                                       AllSuccsCache &AllSuccessors);
 
-  void FindCycleSinkCandidates(MachineCycle *Cycle, MachineBasicBlock *BB,
-                               SmallVectorImpl<MachineInstr *> &Candidates);
-
   bool
   aggressivelySinkIntoCycle(MachineCycle *Cycle, MachineInstr &I,
                             DenseMap<SinkItem, MachineInstr *> &SunkInstrs);
@@ -694,65 +692,6 @@ bool MachineSinking::AllUsesDominatedByBlock(Register Reg,
   return true;
 }
 
-/// Return true if this machine instruction loads from global offset table or
-/// constant pool.
-static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI) {
-  assert(MI.mayLoad() && "Expected MI that loads!");
-
-  // If we lost memory operands, conservatively assume that the instruction
-  // reads from everything..
-  if (MI.memoperands_empty())
-    return true;
-
-  for (MachineMemOperand *MemOp : MI.memoperands())
-    if (const PseudoSourceValue *PSV = MemOp->getPseudoValue())
-      if (PSV->isGOT() || PSV->isConstantPool())
-        return true;
-
-  return false;
-}
-
-void MachineSinking::FindCycleSinkCandidates(
-    MachineCycle *Cycle, MachineBasicBlock *BB,
-    SmallVectorImpl<MachineInstr *> &Candidates) {
-  for (auto &MI : *BB) {
-    LLVM_DEBUG(dbgs() << "CycleSink: Analysing candidate: " << MI);
-    if (MI.isMetaInstruction()) {
-      LLVM_DEBUG(dbgs() << "CycleSink: not sinking meta instruction\n");
-      continue;
-    }
-    if (!TII->shouldSink(MI)) {
-      LLVM_DEBUG(dbgs() << "CycleSink: Instruction not a candidate for this "
-                           "target\n");
-      continue;
-    }
-    if (!isCycleInvariant(Cycle, MI)) {
-      LLVM_DEBUG(dbgs() << "CycleSink: Instruction is not cycle invariant\n");
-      continue;
-    }
-    bool DontMoveAcrossStore = true;
-    if (!MI.isSafeToMove(DontMoveAcrossStore)) {
-      LLVM_DEBUG(dbgs() << "CycleSink: Instruction not safe to move.\n");
-      continue;
-    }
-    if (MI.mayLoad() && !mayLoadFromGOTOrConstantPool(MI)) {
-      LLVM_DEBUG(dbgs() << "CycleSink: Dont sink GOT or constant pool loads\n");
-      continue;
-    }
-    if (MI.isConvergent())
-      continue;
-
-    const MachineOperand &MO = MI.getOperand(0);
-    if (!MO.isReg() || !MO.getReg() || !MO.isDef())
-      continue;
-    if (!MRI->hasOneDef(MO.getReg()))
-      continue;
-
-    LLVM_DEBUG(dbgs() << "CycleSink: Instruction added as candidate.\n");
-    Candidates.push_back(&MI);
-  }
-}
-
 PreservedAnalyses
 MachineSinkingPass::run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MFAM) {
@@ -892,7 +831,9 @@ bool MachineSinking::run(MachineFunction &MF) {
           continue;
         }
         SmallVector<MachineInstr *, 8> Candidates;
-        FindCycleSinkCandidates(Cycle, Preheader, Candidates);
+        for (auto &MI : *Preheader)
+          if (isSinkIntoCycleCandidate(MI, Cycle, MRI, TII))
+            Candidates.push_back(&MI);
 
         unsigned i = 0;
 

>From 9ffa46a073111b09fe95ce8a1aaf7ced4d5e8377 Mon Sep 17 00:00:00 2001
From: dianqk <dianqk at dianqk.net>
Date: Sun, 14 Sep 2025 19:53:35 +0800
Subject: [PATCH 2/2] [MachineSink][Experiment] Rematerialize instructions that
 may be hoisted in LICM

---
 llvm/lib/CodeGen/MachineLICM.cpp |   2 +-
 llvm/lib/CodeGen/MachineSink.cpp | 134 +++++++++++++++++++++++++++----
 2 files changed, 118 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 804a40612e85d..7300f092edacd 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -102,7 +102,7 @@ static cl::opt<bool> SinkInstsIntoCycleBeforeLICM(
     "sink-insts-before-licm",
     cl::desc("Sink instructions into cycles to avoid "
              "register spills"),
-    cl::init(true), cl::Hidden);
+    cl::init(false), cl::Hidden);
 
 STATISTIC(NumHoisted,
           "Number of machine instructions hoisted out of loops");
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 615edf6b414da..2f2de3b42c7f9 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -106,7 +106,13 @@ static cl::opt<bool>
     SinkInstsIntoCycle("sink-insts-to-avoid-spills",
                        cl::desc("Sink instructions into cycles to avoid "
                                 "register spills"),
-                       cl::init(false), cl::Hidden);
+                       cl::init(true), cl::Hidden);
+
+static cl::opt<bool> AggressivelySinkInstsIntoCycle(
+    "aggressively-sink-insts-to-avoid-spills",
+    cl::desc("Aggressively sink instructions into cycles to avoid "
+             "register spills"),
+    cl::init(false), cl::Hidden);
 
 static cl::opt<unsigned> SinkIntoCycleLimit(
     "machine-sink-cycle-limit",
@@ -263,6 +269,8 @@ class MachineSinking {
   aggressivelySinkIntoCycle(MachineCycle *Cycle, MachineInstr &I,
                             DenseMap<SinkItem, MachineInstr *> &SunkInstrs);
 
+  bool rematerializeIntoCycle(MachineCycle *Cycle, MachineInstr &I);
+
   bool isProfitableToSinkTo(Register Reg, MachineInstr &MI,
                             MachineBasicBlock *MBB,
                             MachineBasicBlock *SuccToSinkTo,
@@ -815,21 +823,28 @@ bool MachineSinking::run(MachineFunction &MF) {
   if (SinkInstsIntoCycle) {
     SmallVector<MachineCycle *, 8> Cycles(CI->toplevel_cycles());
     SchedModel.init(STI);
-    bool HasHighPressure;
 
     DenseMap<SinkItem, MachineInstr *> SunkInstrs;
 
-    enum CycleSinkStage { COPY, LOW_LATENCY, AGGRESSIVE, END };
-    for (unsigned Stage = CycleSinkStage::COPY; Stage != CycleSinkStage::END;
-         ++Stage, SunkInstrs.clear()) {
-      HasHighPressure = false;
+    enum CycleSinkStage {
+      COPY,
+      LOW_LATENCY,
+      REMATERIALIZATION,
+      AGGRESSIVE,
+      END
+    };
+    for (auto *Cycle : Cycles) {
+      MachineBasicBlock *Preheader = Cycle->getCyclePreheader();
+      if (!Preheader) {
+        LLVM_DEBUG(dbgs() << "CycleSink: Can't find preheader\n");
+        continue;
+      }
+      bool HasHighPressure = registerPressureExceedsLimit(*Preheader);
+      if (!HasHighPressure)
+        continue;
+      for (unsigned Stage = CycleSinkStage::COPY; Stage != CycleSinkStage::END;
+           ++Stage, SunkInstrs.clear()) {
 
-      for (auto *Cycle : Cycles) {
-        MachineBasicBlock *Preheader = Cycle->getCyclePreheader();
-        if (!Preheader) {
-          LLVM_DEBUG(dbgs() << "CycleSink: Can't find preheader\n");
-          continue;
-        }
         SmallVector<MachineInstr *, 8> Candidates;
         for (auto &MI : *Preheader)
           if (isSinkIntoCycleCandidate(MI, Cycle, MRI, TII))
@@ -860,18 +875,23 @@ bool MachineSinking::run(MachineFunction &MF) {
               !TII->hasLowDefLatency(SchedModel, *I, 0))
             continue;
 
-          if (!aggressivelySinkIntoCycle(Cycle, *I, SunkInstrs))
-            continue;
+          if (Stage == CycleSinkStage::AGGRESSIVE &&
+              AggressivelySinkInstsIntoCycle) {
+            if (!aggressivelySinkIntoCycle(Cycle, *I, SunkInstrs))
+              continue;
+          } else {
+            if (!rematerializeIntoCycle(Cycle, *I))
+              continue;
+          }
           EverMadeChange = true;
           ++NumCycleSunk;
         }
 
         // Recalculate the pressure after sinking
+        HasHighPressure = registerPressureExceedsLimit(*Preheader);
         if (!HasHighPressure)
-          HasHighPressure = registerPressureExceedsLimit(*Preheader);
+          break;
       }
-      if (!HasHighPressure)
-        break;
     }
   }
 
@@ -1771,6 +1791,86 @@ bool MachineSinking::aggressivelySinkIntoCycle(
   return true;
 }
 
+/// Rematerialize instructions into cycles,
+/// since LICM in the middle-end hoisted every instructions without considering
+/// register pressure.
+bool MachineSinking::rematerializeIntoCycle(MachineCycle *Cycle,
+                                            MachineInstr &I) {
+  LLVM_DEBUG(dbgs() << "Rematerialization: Finding sink block for: " << I);
+  MachineBasicBlock *Preheader = Cycle->getCyclePreheader();
+  assert(Preheader && "Cycle sink needs a preheader block");
+  MachineBasicBlock *SinkBlock = nullptr;
+  const MachineOperand &MO = I.getOperand(0);
+  for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) {
+    LLVM_DEBUG(dbgs() << "Rematerialization:   Analysing use: " << MI);
+    if (!Cycle->contains(MI.getParent())) {
+      LLVM_DEBUG(
+          dbgs() << "Rematerialization:   Use not in cycle, can't sink.\n");
+      return false;
+    }
+    if (!SinkBlock) {
+      SinkBlock = MI.getParent();
+      LLVM_DEBUG(dbgs() << "Rematerialization:   Setting sink block to: "
+                        << printMBBReference(*SinkBlock) << "\n");
+      continue;
+    }
+    if (MI.isPHI()) {
+      for (unsigned I = 1; I != MI.getNumOperands(); I += 2) {
+        Register SrcReg = MI.getOperand(I).getReg();
+        if (TRI->regsOverlap(SrcReg, MO.getReg())) {
+          MachineBasicBlock *SrcBB = MI.getOperand(I + 1).getMBB();
+          if (SrcBB != SinkBlock) {
+            SinkBlock = DT->findNearestCommonDominator(SinkBlock, SrcBB);
+            if (!SinkBlock)
+              break;
+          }
+        }
+      }
+    } else {
+      SinkBlock = DT->findNearestCommonDominator(SinkBlock, MI.getParent());
+    }
+    if (!SinkBlock) {
+      LLVM_DEBUG(
+          dbgs() << "Rematerialization:   Can't find nearest dominator\n");
+      return false;
+    }
+    LLVM_DEBUG(
+        dbgs() << "Rematerialization:   Setting nearest common dom block: "
+               << printMBBReference(*SinkBlock) << "\n");
+  }
+  if (!SinkBlock) {
+    LLVM_DEBUG(
+        dbgs() << "Rematerialization: Not sinking, can't find sink block.\n");
+    return false;
+  }
+  if (SinkBlock == Preheader) {
+    LLVM_DEBUG(
+        dbgs()
+        << "Rematerialization: Not sinking, sink block is the preheader\n");
+    return false;
+  }
+  for (MachineInstr &MI : MRI->use_instructions(MO.getReg())) {
+    if (MI.isPHI() && MI.getParent() == SinkBlock) {
+      LLVM_DEBUG(dbgs() << "Rematerialization: Not sinking, sink block is "
+                           "using it on PHI.\n");
+      return false;
+    }
+  }
+  LLVM_DEBUG(dbgs() << "Rematerialization: Sinking instruction!\n");
+  SinkBlock->splice(SinkBlock->SkipPHIsAndLabels(SinkBlock->begin()), Preheader,
+                    I);
+  // Conservatively clear any kill flags on uses of sunk instruction
+  for (MachineOperand &MO : I.operands()) {
+    if (MO.isReg() && MO.readsReg())
+      MRI->clearKillFlags(MO.getReg());
+  }
+  // The instruction is moved from its basic block, so do not retain the
+  // debug information.
+  assert(!I.isDebugInstr() && "Should not sink debug inst");
+  I.setDebugLoc(DebugLoc());
+  return true;
+}
+
 /// SinkInstruction - Determine whether it is safe to sink the specified machine
 /// instruction out of its current block into a successor.
 bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,



More information about the llvm-commits mailing list