<div dir="ltr">Wei,<div><br></div><div>It seems that memory problem still exist since it breaks build bot:</div><div><br></div><div><a href="http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-bootstrap/builds/11394">http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-bootstrap/builds/11394</a><br></div><div><br></div><div>Can you take a look?</div><div><br></div><div><div>==19290==WARNING: MemorySanitizer: use-of-uninitialized-value</div><div> #0 0x8d4c78 in AdvanceIfNotValid /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/include/llvm/ADT/SmallPtrSet.h:224:62</div><div> #1 0x8d4c78 in SmallPtrSetIteratorImpl /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/include/llvm/ADT/SmallPtrSet.h:207</div><div> #2 0x8d4c78 in SmallPtrSetIterator /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/include/llvm/ADT/SmallPtrSet.h:243</div><div> #3 0x8d4c78 in llvm::SmallPtrSetImpl<llvm::DomTreeNodeBase<llvm::MachineBasicBlock>*>::insert(llvm::DomTreeNodeBase<llvm::MachineBasicBlock>*) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/include/llvm/ADT/SmallPtrSet.h:319</div><div> #4 0x364fdad in insert<llvm::SmallPtrSetIterator<llvm::DomTreeNodeBase<llvm::MachineBasicBlock> *> > /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/include/llvm/ADT/SmallPtrSet.h:336:7</div><div> #5 0x364fdad in runHoistSpills /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/CodeGen/InlineSpiller.cpp:1276</div><div> #6 0x364fdad in (anonymous namespace)::HoistSpillHelper::hoistAllSpills(llvm::LiveRangeEdit&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/CodeGen/InlineSpiller.cpp:1384</div><div> #7 0x3634bf3 in (anonymous namespace)::InlineSpiller::postOptimization() /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/CodeGen/InlineSpiller.cpp:1044:3</div><div> #8 0x370e3f0 in llvm::RegAllocBase::postOptimization() /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/CodeGen/RegAllocBase.cpp:158:3</div><div> #9 0x33624e9 in (anonymous namespace)::RAGreedy::runOnMachineFunction(llvm::MachineFunction&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/CodeGen/RegAllocGreedy.cpp:2613:3</div><div> #10 0x31be092 in llvm::MachineFunctionPass::runOnFunction(llvm::Function&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/CodeGen/MachineFunctionPass.cpp:60:13</div><div> #11 0x39b375d in llvm::FPPassManager::runOnFunction(llvm::Function&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/IR/LegacyPassManager.cpp:1550:23</div><div> #12 0x39b3e03 in llvm::FPPassManager::runOnModule(llvm::Module&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/IR/LegacyPassManager.cpp:1571:16</div><div> #13 0x39b5122 in runOnModule /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/IR/LegacyPassManager.cpp:1627:23</div><div> #14 0x39b5122 in llvm::legacy::PassManagerImpl::run(llvm::Module&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/lib/IR/LegacyPassManager.cpp:1730</div><div> #15 0x4bfef6f in (anonymous namespace)::EmitAssemblyHelper::EmitAssembly(clang::BackendAction, llvm::raw_pwrite_stream*) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp:707:5</div><div> #16 0x4bf5cef in clang::EmitBackendOutput(clang::DiagnosticsEngine&, clang::CodeGenOptions const&, clang::TargetOptions const&, clang::LangOptions const&, llvm::DataLayout const&, llvm::Module*, clang::BackendAction, llvm::raw_pwrite_stream*) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp:719:3</div><div> #17 0x6455836 in clang::BackendConsumer::HandleTranslationUnit(clang::ASTContext&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp:176:7</div><div> #18 0x7191e3b in clang::ParseAST(clang::Sema&, bool, bool) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/Parse/ParseAST.cpp:168:3</div><div> #19 0x5902b97 in clang::FrontendAction::Execute() /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/Frontend/FrontendAction.cpp:457:8</div><div> #20 0x5861214 in clang::CompilerInstance::ExecuteAction(clang::FrontendAction&) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/Frontend/CompilerInstance.cpp:873:7</div><div> #21 0x5b0e686 in clang::ExecuteCompilerInvocation(clang::CompilerInstance*) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp:241:18</div><div> #22 0x83438f in cc1_main(llvm::ArrayRef<char const*>, char const*, void*) /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/tools/driver/cc1_main.cpp:116:13</div><div> #23 0x82fc48 in ExecuteCC1Tool /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/tools/driver/driver.cpp:301:12</div><div> #24 0x82fc48 in main /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/tools/clang/tools/driver/driver.cpp:366</div><div> #25 0x7fb71087eec4 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21ec4)</div><div> #26 0x7a557d in _start (/mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm_build_msan/bin/clang-3.9+0x7a557d)</div><div><br></div><div>SUMMARY: MemorySanitizer: use-of-uninitialized-value /mnt/b/sanitizer-buildbot2/sanitizer-x86_64-linux-bootstrap/build/llvm/include/llvm/ADT/SmallPtrSet.h:224:62 in AdvanceIfNotValid</div></div><div><br></div></div><br><div class="gmail_quote"><div dir="ltr">On Wed, Apr 6, 2016 at 8:46 AM Wei Mi via llvm-commits <<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: wmi<br>
Date: Wed Apr 6 10:41:07 2016<br>
New Revision: 265547<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=265547&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=265547&view=rev</a><br>
Log:<br>
Recommit r265309 after fixed an invalid memory reference bug happened<br>
when DenseMap growed and moved memory. I verified it fixed the bootstrap<br>
problem on x86_64-linux-gnu but I cannot verify whether it fixes<br>
the bootstrap error on clang-ppc64be-linux. I will watch the build-bot<br>
result closely.<br>
<br>
Replace analyzeSiblingValues with new algorithm to fix its compile<br>
time issue. The patch is to solve PR17409 and its duplicates.<br>
<br>
analyzeSiblingValues is a N x N complexity algorithm where N is<br>
the number of siblings generated by reg splitting. Although it<br>
causes siginificant compile time issue when N is large, it is also<br>
important for performance since it removes redundent spills and<br>
enables rematerialization.<br>
<br>
To solve the compile time issue, the patch removes analyzeSiblingValues<br>
and replaces it with lower cost alternatives containing two parts. The<br>
first part creates a new spill hoisting method in postOptimization of<br>
register allocation. It does spill hoisting at once after all the spills<br>
are generated instead of inside every instance of selectOrSplit. The<br>
second part queries the define expr of the original register for<br>
rematerializaiton and keep it always available during register allocation<br>
even if it is already dead. It deletes those dead instructions only in<br>
postOptimization. With the two parts in the patch, it can remove<br>
analyzeSiblingValues without sacrificing performance.<br>
<br>
Differential Revision: <a href="http://reviews.llvm.org/D15302" rel="noreferrer" target="_blank">http://reviews.llvm.org/D15302</a><br>
<br>
Added:<br>
llvm/trunk/test/CodeGen/X86/hoist-spill.ll<br>
llvm/trunk/test/CodeGen/X86/new-remat.ll<br>
Removed:<br>
llvm/trunk/test/CodeGen/AArch64/aarch64-deferred-spilling.ll<br>
Modified:<br>
llvm/trunk/include/llvm/CodeGen/LiveRangeEdit.h<br>
llvm/trunk/lib/CodeGen/InlineSpiller.cpp<br>
llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp<br>
llvm/trunk/lib/CodeGen/RegAllocBase.cpp<br>
llvm/trunk/lib/CodeGen/RegAllocBase.h<br>
llvm/trunk/lib/CodeGen/RegAllocBasic.cpp<br>
llvm/trunk/lib/CodeGen/RegAllocGreedy.cpp<br>
llvm/trunk/lib/CodeGen/RegAllocPBQP.cpp<br>
llvm/trunk/lib/CodeGen/Spiller.h<br>
llvm/trunk/lib/CodeGen/SplitKit.cpp<br>
llvm/trunk/lib/CodeGen/SplitKit.h<br>
llvm/trunk/test/CodeGen/X86/fp128-compare.ll<br>
llvm/trunk/test/CodeGen/X86/ragreedy-hoist-spill.ll<br>
<br>
Modified: llvm/trunk/include/llvm/CodeGen/LiveRangeEdit.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/LiveRangeEdit.h?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/LiveRangeEdit.h?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/LiveRangeEdit.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/LiveRangeEdit.h Wed Apr 6 10:41:07 2016<br>
@@ -72,6 +72,10 @@ private:<br>
/// ScannedRemattable - true when remattable values have been identified.<br>
bool ScannedRemattable;<br>
<br>
+ /// DeadRemats - The saved instructions which have already been dead after<br>
+ /// rematerialization but not deleted yet -- to be done in postOptimization.<br>
+ SmallPtrSet<MachineInstr *, 32> *DeadRemats;<br>
+<br>
/// Remattable - Values defined by remattable instructions as identified by<br>
/// tii.isTriviallyReMaterializable().<br>
SmallPtrSet<const VNInfo*,4> Remattable;<br>
@@ -116,13 +120,16 @@ public:<br>
/// @param vrm Map of virtual registers to physical registers for this<br>
/// function. If NULL, no virtual register map updates will<br>
/// be done. This could be the case if called before Regalloc.<br>
+ /// @param deadRemats The collection of all the instructions defining an<br>
+ /// original reg and are dead after remat.<br>
LiveRangeEdit(LiveInterval *parent, SmallVectorImpl<unsigned> &newRegs,<br>
MachineFunction &MF, LiveIntervals &lis, VirtRegMap *vrm,<br>
- Delegate *delegate = nullptr)<br>
+ Delegate *delegate = nullptr,<br>
+ SmallPtrSet<MachineInstr *, 32> *deadRemats = nullptr)<br>
: Parent(parent), NewRegs(newRegs), MRI(MF.getRegInfo()), LIS(lis),<br>
- VRM(vrm), TII(*MF.getSubtarget().getInstrInfo()),<br>
- TheDelegate(delegate), FirstNew(newRegs.size()),<br>
- ScannedRemattable(false) {<br>
+ VRM(vrm), TII(*MF.getSubtarget().getInstrInfo()), TheDelegate(delegate),<br>
+ FirstNew(newRegs.size()), ScannedRemattable(false),<br>
+ DeadRemats(deadRemats) {<br>
MRI.setDelegate(this);<br>
}<br>
<br>
@@ -142,6 +149,16 @@ public:<br>
bool empty() const { return size() == 0; }<br>
unsigned get(unsigned idx) const { return NewRegs[idx+FirstNew]; }<br>
<br>
+ /// pop_back - It allows LiveRangeEdit users to drop new registers.<br>
+ /// The context is when an original def instruction of a register is<br>
+ /// dead after rematerialization, we still want to keep it for following<br>
+ /// rematerializations. We save the def instruction in DeadRemats,<br>
+ /// and replace the original dst register with a new dummy register so<br>
+ /// the live range of original dst register can be shrinked normally.<br>
+ /// We don't want to allocate phys register for the dummy register, so<br>
+ /// we want to drop it from the NewRegs set.<br>
+ void pop_back() { NewRegs.pop_back(); }<br>
+<br>
ArrayRef<unsigned> regs() const {<br>
return makeArrayRef(NewRegs).slice(FirstNew);<br>
}<br>
@@ -175,15 +192,15 @@ public:<br>
/// Remat - Information needed to rematerialize at a specific location.<br>
struct Remat {<br>
VNInfo *ParentVNI; // parent_'s value at the remat location.<br>
- MachineInstr *OrigMI; // Instruction defining ParentVNI.<br>
+ MachineInstr *OrigMI; // Instruction defining OrigVNI. It contains the<br>
+ // real expr for remat.<br>
explicit Remat(VNInfo *ParentVNI) : ParentVNI(ParentVNI), OrigMI(nullptr) {}<br>
};<br>
<br>
/// canRematerializeAt - Determine if ParentVNI can be rematerialized at<br>
/// UseIdx. It is assumed that parent_.getVNINfoAt(UseIdx) == ParentVNI.<br>
/// When cheapAsAMove is set, only cheap remats are allowed.<br>
- bool canRematerializeAt(Remat &RM,<br>
- SlotIndex UseIdx,<br>
+ bool canRematerializeAt(Remat &RM, VNInfo *OrigVNI, SlotIndex UseIdx,<br>
bool cheapAsAMove);<br>
<br>
/// rematerializeAt - Rematerialize RM.ParentVNI into DestReg by inserting an<br>
@@ -208,6 +225,12 @@ public:<br>
return Rematted.count(ParentVNI);<br>
}<br>
<br>
+ void markDeadRemat(MachineInstr *inst) {<br>
+ // DeadRemats is an optional field.<br>
+ if (DeadRemats)<br>
+ DeadRemats->insert(inst);<br>
+ }<br>
+<br>
/// eraseVirtReg - Notify the delegate that Reg is no longer in use, and try<br>
/// to erase it from LIS.<br>
void eraseVirtReg(unsigned Reg);<br>
@@ -218,8 +241,11 @@ public:<br>
/// RegsBeingSpilled lists registers currently being spilled by the register<br>
/// allocator. These registers should not be split into new intervals<br>
/// as currently those new intervals are not guaranteed to spill.<br>
- void eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,<br>
- ArrayRef<unsigned> RegsBeingSpilled = None);<br>
+ /// NoSplit indicates this func is used after the iterations of selectOrSplit<br>
+ /// where registers should not be split into new intervals.<br>
+ void eliminateDeadDefs(SmallVectorImpl<MachineInstr *> &Dead,<br>
+ ArrayRef<unsigned> RegsBeingSpilled = None,<br>
+ bool NoSplit = false);<br>
<br>
/// calculateRegClassAndHint - Recompute register class and hint for each new<br>
/// register.<br>
<br>
Modified: llvm/trunk/lib/CodeGen/InlineSpiller.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/InlineSpiller.cpp?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/InlineSpiller.cpp?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/InlineSpiller.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/InlineSpiller.cpp Wed Apr 6 10:41:07 2016<br>
@@ -48,13 +48,77 @@ STATISTIC(NumReloadsRemoved, "Number of<br>
STATISTIC(NumFolded, "Number of folded stack accesses");<br>
STATISTIC(NumFoldedLoads, "Number of folded loads");<br>
STATISTIC(NumRemats, "Number of rematerialized defs for spilling");<br>
-STATISTIC(NumOmitReloadSpill, "Number of omitted spills of reloads");<br>
-STATISTIC(NumHoists, "Number of hoisted spills");<br>
<br>
static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden,<br>
cl::desc("Disable inline spill hoisting"));<br>
<br>
namespace {<br>
+class HoistSpillHelper {<br>
+ LiveIntervals &LIS;<br>
+ LiveStacks &LSS;<br>
+ AliasAnalysis *AA;<br>
+ MachineDominatorTree &MDT;<br>
+ MachineLoopInfo &Loops;<br>
+ VirtRegMap &VRM;<br>
+ MachineFrameInfo &MFI;<br>
+ MachineRegisterInfo &MRI;<br>
+ const TargetInstrInfo &TII;<br>
+ const TargetRegisterInfo &TRI;<br>
+ const MachineBlockFrequencyInfo &MBFI;<br>
+<br>
+ // Map from StackSlot to its original register.<br>
+ DenseMap<int, unsigned> StackSlotToReg;<br>
+ // Map from pair of (StackSlot and Original VNI) to a set of spills which<br>
+ // have the same stackslot and have equal values defined by Original VNI.<br>
+ // These spills are mergeable and are hoist candiates.<br>
+ typedef DenseMap<std::pair<int, VNInfo *>, SmallPtrSet<MachineInstr *, 16>><br>
+ MergeableSpillsMap;<br>
+ MergeableSpillsMap MergeableSpills;<br>
+<br>
+ /// This is the map from original register to a set containing all its<br>
+ /// siblings. To hoist a spill to another BB, we need to find out a live<br>
+ /// sibling there and use it as the source of the new spill.<br>
+ DenseMap<unsigned, SmallSetVector<unsigned, 16>> Virt2SiblingsMap;<br>
+<br>
+ bool isSpillCandBB(unsigned OrigReg, VNInfo &OrigVNI, MachineBasicBlock &BB,<br>
+ unsigned &LiveReg);<br>
+<br>
+ void rmRedundantSpills(<br>
+ SmallPtrSet<MachineInstr *, 16> &Spills,<br>
+ SmallVectorImpl<MachineInstr *> &SpillsToRm,<br>
+ DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill);<br>
+<br>
+ void getVisitOrders(<br>
+ MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills,<br>
+ SmallVectorImpl<MachineDomTreeNode *> &Orders,<br>
+ SmallVectorImpl<MachineInstr *> &SpillsToRm,<br>
+ DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep,<br>
+ DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill);<br>
+<br>
+ void runHoistSpills(unsigned OrigReg, VNInfo &OrigVNI,<br>
+ SmallPtrSet<MachineInstr *, 16> &Spills,<br>
+ SmallVectorImpl<MachineInstr *> &SpillsToRm,<br>
+ DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns);<br>
+<br>
+public:<br>
+ HoistSpillHelper(MachineFunctionPass &pass, MachineFunction &mf,<br>
+ VirtRegMap &vrm)<br>
+ : LIS(pass.getAnalysis<LiveIntervals>()),<br>
+ LSS(pass.getAnalysis<LiveStacks>()),<br>
+ AA(&pass.getAnalysis<AAResultsWrapperPass>().getAAResults()),<br>
+ MDT(pass.getAnalysis<MachineDominatorTree>()),<br>
+ Loops(pass.getAnalysis<MachineLoopInfo>()), VRM(vrm),<br>
+ MFI(*mf.getFrameInfo()), MRI(mf.getRegInfo()),<br>
+ TII(*mf.getSubtarget().getInstrInfo()),<br>
+ TRI(*mf.getSubtarget().getRegisterInfo()),<br>
+ MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()) {}<br>
+<br>
+ void addToMergeableSpills(MachineInstr *Spill, int StackSlot,<br>
+ unsigned Original);<br>
+ bool rmFromMergeableSpills(MachineInstr *Spill, int StackSlot);<br>
+ void hoistAllSpills(LiveRangeEdit &Edit);<br>
+};<br>
+<br>
class InlineSpiller : public Spiller {<br>
MachineFunction &MF;<br>
LiveIntervals &LIS;<br>
@@ -85,56 +149,12 @@ class InlineSpiller : public Spiller {<br>
// Values that failed to remat at some point.<br>
SmallPtrSet<VNInfo*, 8> UsedValues;<br>
<br>
-public:<br>
- // Information about a value that was defined by a copy from a sibling<br>
- // register.<br>
- struct SibValueInfo {<br>
- // True when all reaching defs were reloads: No spill is necessary.<br>
- bool AllDefsAreReloads;<br>
-<br>
- // True when value is defined by an original PHI not from splitting.<br>
- bool DefByOrigPHI;<br>
-<br>
- // True when the COPY defining this value killed its source.<br>
- bool KillsSource;<br>
-<br>
- // The preferred register to spill.<br>
- unsigned SpillReg;<br>
-<br>
- // The value of SpillReg that should be spilled.<br>
- VNInfo *SpillVNI;<br>
-<br>
- // The block where SpillVNI should be spilled. Currently, this must be the<br>
- // block containing SpillVNI->def.<br>
- MachineBasicBlock *SpillMBB;<br>
-<br>
- // A defining instruction that is not a sibling copy or a reload, or NULL.<br>
- // This can be used as a template for rematerialization.<br>
- MachineInstr *DefMI;<br>
-<br>
- // List of values that depend on this one. These values are actually the<br>
- // same, but live range splitting has placed them in different registers,<br>
- // or SSA update needed to insert PHI-defs to preserve SSA form. This is<br>
- // copies of the current value and phi-kills. Usually only phi-kills cause<br>
- // more than one dependent value.<br>
- TinyPtrVector<VNInfo*> Deps;<br>
-<br>
- SibValueInfo(unsigned Reg, VNInfo *VNI)<br>
- : AllDefsAreReloads(true), DefByOrigPHI(false), KillsSource(false),<br>
- SpillReg(Reg), SpillVNI(VNI), SpillMBB(nullptr), DefMI(nullptr) {}<br>
-<br>
- // Returns true when a def has been found.<br>
- bool hasDef() const { return DefByOrigPHI || DefMI; }<br>
- };<br>
-<br>
-private:<br>
- // Values in RegsToSpill defined by sibling copies.<br>
- typedef DenseMap<VNInfo*, SibValueInfo> SibValueMap;<br>
- SibValueMap SibValues;<br>
-<br>
// Dead defs generated during spilling.<br>
SmallVector<MachineInstr*, 8> DeadDefs;<br>
<br>
+ // Object records spills information and does the hoisting.<br>
+ HoistSpillHelper HSpiller;<br>
+<br>
~InlineSpiller() override {}<br>
<br>
public:<br>
@@ -147,9 +167,11 @@ public:<br>
MFI(*mf.getFrameInfo()), MRI(mf.getRegInfo()),<br>
TII(*mf.getSubtarget().getInstrInfo()),<br>
TRI(*mf.getSubtarget().getRegisterInfo()),<br>
- MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()) {}<br>
+ MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()),<br>
+ HSpiller(pass, mf, vrm) {}<br>
<br>
void spill(LiveRangeEdit &) override;<br>
+ void postOptimization() override;<br>
<br>
private:<br>
bool isSnippet(const LiveInterval &SnipLI);<br>
@@ -161,11 +183,7 @@ private:<br>
}<br>
<br>
bool isSibling(unsigned Reg);<br>
- MachineInstr *traceSiblingValue(unsigned, VNInfo*, VNInfo*);<br>
- void propagateSiblingValue(SibValueMap::iterator, VNInfo *VNI = nullptr);<br>
- void analyzeSiblingValues();<br>
-<br>
- bool hoistSpill(LiveInterval &SpillLI, MachineInstr &CopyMI);<br>
+ bool hoistSpillInsideBB(LiveInterval &SpillLI, MachineInstr &CopyMI);<br>
void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI);<br>
<br>
void markValueUsed(LiveInterval*, VNInfo*);<br>
@@ -297,417 +315,45 @@ void InlineSpiller::collectRegsToSpill()<br>
}<br>
}<br>
<br>
-<br>
-//===----------------------------------------------------------------------===//<br>
-// Sibling Values<br>
-//===----------------------------------------------------------------------===//<br>
-<br>
-// After live range splitting, some values to be spilled may be defined by<br>
-// copies from sibling registers. We trace the sibling copies back to the<br>
-// original value if it still exists. We need it for rematerialization.<br>
-//<br>
-// Even when the value can't be rematerialized, we still want to determine if<br>
-// the value has already been spilled, or we may want to hoist the spill from a<br>
-// loop.<br>
-<br>
bool InlineSpiller::isSibling(unsigned Reg) {<br>
return TargetRegisterInfo::isVirtualRegister(Reg) &&<br>
VRM.getOriginal(Reg) == Original;<br>
}<br>
<br>
-#ifndef NDEBUG<br>
-static raw_ostream &operator<<(raw_ostream &OS,<br>
- const InlineSpiller::SibValueInfo &SVI) {<br>
- OS << "spill " << PrintReg(SVI.SpillReg) << ':'<br>
- << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def;<br>
- if (SVI.SpillMBB)<br>
- OS << " in BB#" << SVI.SpillMBB->getNumber();<br>
- if (SVI.AllDefsAreReloads)<br>
- OS << " all-reloads";<br>
- if (SVI.DefByOrigPHI)<br>
- OS << " orig-phi";<br>
- if (SVI.KillsSource)<br>
- OS << " kill";<br>
- OS << " deps[";<br>
- for (VNInfo *Dep : SVI.Deps)<br>
- OS << ' ' << Dep->id << '@' << Dep->def;<br>
- OS << " ]";<br>
- if (SVI.DefMI)<br>
- OS << " def: " << *SVI.DefMI;<br>
- else<br>
- OS << '\n';<br>
- return OS;<br>
-}<br>
-#endif<br>
-<br>
-/// propagateSiblingValue - Propagate the value in SVI to dependents if it is<br>
-/// known. Otherwise remember the dependency for later.<br>
+/// It is beneficial to spill to earlier place in the same BB in case<br>
+/// as follows:<br>
+/// There is an alternative def earlier in the same MBB.<br>
+/// Hoist the spill as far as possible in SpillMBB. This can ease<br>
+/// register pressure:<br>
///<br>
-/// @param SVIIter SibValues entry to propagate.<br>
-/// @param VNI Dependent value, or NULL to propagate to all saved dependents.<br>
-void InlineSpiller::propagateSiblingValue(SibValueMap::iterator SVIIter,<br>
- VNInfo *VNI) {<br>
- SibValueMap::value_type *SVI = &*SVIIter;<br>
-<br>
- // When VNI is non-NULL, add it to SVI's deps, and only propagate to that.<br>
- TinyPtrVector<VNInfo*> FirstDeps;<br>
- if (VNI) {<br>
- FirstDeps.push_back(VNI);<br>
- SVI->second.Deps.push_back(VNI);<br>
- }<br>
-<br>
- // Has the value been completely determined yet? If not, defer propagation.<br>
- if (!SVI->second.hasDef())<br>
- return;<br>
-<br>
- // Work list of values to propagate.<br>
- SmallSetVector<SibValueMap::value_type *, 8> WorkList;<br>
- WorkList.insert(SVI);<br>
-<br>
- do {<br>
- SVI = WorkList.pop_back_val();<br>
- TinyPtrVector<VNInfo*> *Deps = VNI ? &FirstDeps : &SVI->second.Deps;<br>
- VNI = nullptr;<br>
-<br>
- SibValueInfo &SV = SVI->second;<br>
- if (!SV.SpillMBB)<br>
- SV.SpillMBB = LIS.getMBBFromIndex(SV.SpillVNI->def);<br>
-<br>
- DEBUG(dbgs() << " prop to " << Deps->size() << ": "<br>
- << SVI->first->id << '@' << SVI->first->def << ":\t" << SV);<br>
-<br>
- assert(SV.hasDef() && "Propagating undefined value");<br>
-<br>
- // Should this value be propagated as a preferred spill candidate? We don't<br>
- // propagate values of registers that are about to spill.<br>
- bool PropSpill = !DisableHoisting && !isRegToSpill(SV.SpillReg);<br>
- unsigned SpillDepth = ~0u;<br>
-<br>
- for (VNInfo *Dep : *Deps) {<br>
- SibValueMap::iterator DepSVI = SibValues.find(Dep);<br>
- assert(DepSVI != SibValues.end() && "Dependent value not in SibValues");<br>
- SibValueInfo &DepSV = DepSVI->second;<br>
- if (!DepSV.SpillMBB)<br>
- DepSV.SpillMBB = LIS.getMBBFromIndex(DepSV.SpillVNI->def);<br>
-<br>
- bool Changed = false;<br>
-<br>
- // Propagate defining instruction.<br>
- if (!DepSV.hasDef()) {<br>
- Changed = true;<br>
- DepSV.DefMI = SV.DefMI;<br>
- DepSV.DefByOrigPHI = SV.DefByOrigPHI;<br>
- }<br>
-<br>
- // Propagate AllDefsAreReloads. For PHI values, this computes an AND of<br>
- // all predecessors.<br>
- if (!SV.AllDefsAreReloads && DepSV.AllDefsAreReloads) {<br>
- Changed = true;<br>
- DepSV.AllDefsAreReloads = false;<br>
- }<br>
-<br>
- // Propagate best spill value.<br>
- if (PropSpill && SV.SpillVNI != DepSV.SpillVNI) {<br>
- if (SV.SpillMBB == DepSV.SpillMBB) {<br>
- // DepSV is in the same block. Hoist when dominated.<br>
- if (DepSV.KillsSource && SV.SpillVNI->def < DepSV.SpillVNI->def) {<br>
- // This is an alternative def earlier in the same MBB.<br>
- // Hoist the spill as far as possible in SpillMBB. This can ease<br>
- // register pressure:<br>
- //<br>
- // x = def<br>
- // y = use x<br>
- // s = copy x<br>
- //<br>
- // Hoisting the spill of s to immediately after the def removes the<br>
- // interference between x and y:<br>
- //<br>
- // x = def<br>
- // spill x<br>
- // y = use x<kill><br>
- //<br>
- // This hoist only helps when the DepSV copy kills its source.<br>
- Changed = true;<br>
- DepSV.SpillReg = SV.SpillReg;<br>
- DepSV.SpillVNI = SV.SpillVNI;<br>
- DepSV.SpillMBB = SV.SpillMBB;<br>
- }<br>
- } else {<br>
- // DepSV is in a different block.<br>
- if (SpillDepth == ~0u)<br>
- SpillDepth = Loops.getLoopDepth(SV.SpillMBB);<br>
-<br>
- // Also hoist spills to blocks with smaller loop depth, but make sure<br>
- // that the new value dominates. Non-phi dependents are always<br>
- // dominated, phis need checking.<br>
-<br>
- const BranchProbability MarginProb(4, 5); // 80%<br>
- // Hoist a spill to outer loop if there are multiple dependents (it<br>
- // can be beneficial if more than one dependents are hoisted) or<br>
- // if DepSV (the hoisting source) is hotter than SV (the hoisting<br>
- // destination) (we add a 80% margin to bias a little towards<br>
- // loop depth).<br>
- bool HoistCondition =<br>
- (MBFI.getBlockFreq(DepSV.SpillMBB) >=<br>
- (MBFI.getBlockFreq(SV.SpillMBB) * MarginProb)) ||<br>
- Deps->size() > 1;<br>
-<br>
- if ((Loops.getLoopDepth(DepSV.SpillMBB) > SpillDepth) &&<br>
- HoistCondition &&<br>
- (!DepSVI->first->isPHIDef() ||<br>
- MDT.dominates(SV.SpillMBB, DepSV.SpillMBB))) {<br>
- Changed = true;<br>
- DepSV.SpillReg = SV.SpillReg;<br>
- DepSV.SpillVNI = SV.SpillVNI;<br>
- DepSV.SpillMBB = SV.SpillMBB;<br>
- }<br>
- }<br>
- }<br>
-<br>
- if (!Changed)<br>
- continue;<br>
-<br>
- // Something changed in DepSVI. Propagate to dependents.<br>
- WorkList.insert(&*DepSVI);<br>
-<br>
- DEBUG(dbgs() << " update " << DepSVI->first->id << '@'<br>
- << DepSVI->first->def << " to:\t" << DepSV);<br>
- }<br>
- } while (!WorkList.empty());<br>
-}<br>
-<br>
-/// traceSiblingValue - Trace a value that is about to be spilled back to the<br>
-/// real defining instructions by looking through sibling copies. Always stay<br>
-/// within the range of OrigVNI so the registers are known to carry the same<br>
-/// value.<br>
+/// x = def<br>
+/// y = use x<br>
+/// s = copy x<br>
///<br>
-/// Determine if the value is defined by all reloads, so spilling isn't<br>
-/// necessary - the value is already in the stack slot.<br>
+/// Hoisting the spill of s to immediately after the def removes the<br>
+/// interference between x and y:<br>
///<br>
-/// Return a defining instruction that may be a candidate for rematerialization.<br>
+/// x = def<br>
+/// spill x<br>
+/// y = use x<kill><br>
///<br>
-MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,<br>
- VNInfo *OrigVNI) {<br>
- // Check if a cached value already exists.<br>
- SibValueMap::iterator SVI;<br>
- bool Inserted;<br>
- std::tie(SVI, Inserted) =<br>
- SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI)));<br>
- if (!Inserted) {<br>
- DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':'<br>
- << UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second);<br>
- return SVI->second.DefMI;<br>
- }<br>
-<br>
- DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':'<br>
- << UseVNI->id << '@' << UseVNI->def << '\n');<br>
-<br>
- // List of (Reg, VNI) that have been inserted into SibValues, but need to be<br>
- // processed.<br>
- SmallVector<std::pair<unsigned, VNInfo*>, 8> WorkList;<br>
- WorkList.push_back(std::make_pair(UseReg, UseVNI));<br>
-<br>
- LiveInterval &OrigLI = LIS.getInterval(Original);<br>
- do {<br>
- unsigned Reg;<br>
- VNInfo *VNI;<br>
- std::tie(Reg, VNI) = WorkList.pop_back_val();<br>
- DEBUG(dbgs() << " " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def<br>
- << ":\t");<br>
-<br>
- // First check if this value has already been computed.<br>
- SVI = SibValues.find(VNI);<br>
- assert(SVI != SibValues.end() && "Missing SibValues entry");<br>
-<br>
- // Trace through PHI-defs created by live range splitting.<br>
- if (VNI->isPHIDef()) {<br>
- // Stop at original PHIs. We don't know the value at the<br>
- // predecessors. Look up the VNInfo for the current definition<br>
- // in OrigLI, to properly determine whether or not this phi was<br>
- // added by splitting.<br>
- if (VNI->def == OrigLI.getVNInfoAt(VNI->def)->def) {<br>
- DEBUG(dbgs() << "orig phi value\n");<br>
- SVI->second.DefByOrigPHI = true;<br>
- SVI->second.AllDefsAreReloads = false;<br>
- propagateSiblingValue(SVI);<br>
- continue;<br>
- }<br>
-<br>
- // This is a PHI inserted by live range splitting. We could trace the<br>
- // live-out value from predecessor blocks, but that search can be very<br>
- // expensive if there are many predecessors and many more PHIs as<br>
- // generated by tail-dup when it sees an indirectbr. Instead, look at<br>
- // all the non-PHI defs that have the same value as OrigVNI. They must<br>
- // jointly dominate VNI->def. This is not optimal since VNI may actually<br>
- // be jointly dominated by a smaller subset of defs, so there is a change<br>
- // we will miss a AllDefsAreReloads optimization.<br>
-<br>
- // Separate all values dominated by OrigVNI into PHIs and non-PHIs.<br>
- SmallVector<VNInfo*, 8> PHIs, NonPHIs;<br>
- LiveInterval &LI = LIS.getInterval(Reg);<br>
-<br>
- for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end();<br>
- VI != VE; ++VI) {<br>
- VNInfo *VNI2 = *VI;<br>
- if (VNI2->isUnused())<br>
- continue;<br>
- if (!OrigLI.containsOneValue() &&<br>
- OrigLI.getVNInfoAt(VNI2->def) != OrigVNI)<br>
- continue;<br>
- if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def)<br>
- PHIs.push_back(VNI2);<br>
- else<br>
- NonPHIs.push_back(VNI2);<br>
- }<br>
- DEBUG(dbgs() << "split phi value, checking " << PHIs.size()<br>
- << " phi-defs, and " << NonPHIs.size()<br>
- << " non-phi/orig defs\n");<br>
-<br>
- // Create entries for all the PHIs. Don't add them to the worklist, we<br>
- // are processing all of them in one go here.<br>
- for (VNInfo *PHI : PHIs)<br>
- SibValues.insert(std::make_pair(PHI, SibValueInfo(Reg, PHI)));<br>
-<br>
- // Add every PHI as a dependent of all the non-PHIs.<br>
- for (VNInfo *NonPHI : NonPHIs) {<br>
- // Known value? Try an insertion.<br>
- std::tie(SVI, Inserted) =<br>
- SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI)));<br>
- // Add all the PHIs as dependents of NonPHI.<br>
- SVI->second.Deps.insert(SVI->second.Deps.end(), PHIs.begin(),<br>
- PHIs.end());<br>
- // This is the first time we see NonPHI, add it to the worklist.<br>
- if (Inserted)<br>
- WorkList.push_back(std::make_pair(Reg, NonPHI));<br>
- else<br>
- // Propagate to all inserted PHIs, not just VNI.<br>
- propagateSiblingValue(SVI);<br>
- }<br>
-<br>
- // Next work list item.<br>
- continue;<br>
- }<br>
-<br>
- MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);<br>
- assert(MI && "Missing def");<br>
-<br>
- // Trace through sibling copies.<br>
- if (unsigned SrcReg = isFullCopyOf(MI, Reg)) {<br>
- if (isSibling(SrcReg)) {<br>
- LiveInterval &SrcLI = LIS.getInterval(SrcReg);<br>
- LiveQueryResult SrcQ = SrcLI.Query(VNI->def);<br>
- assert(SrcQ.valueIn() && "Copy from non-existing value");<br>
- // Check if this COPY kills its source.<br>
- SVI->second.KillsSource = SrcQ.isKill();<br>
- VNInfo *SrcVNI = SrcQ.valueIn();<br>
- DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':'<br>
- << SrcVNI->id << '@' << SrcVNI->def<br>
- << " kill=" << unsigned(SVI->second.KillsSource) << '\n');<br>
- // Known sibling source value? Try an insertion.<br>
- std::tie(SVI, Inserted) = SibValues.insert(<br>
- std::make_pair(SrcVNI, SibValueInfo(SrcReg, SrcVNI)));<br>
- // This is the first time we see Src, add it to the worklist.<br>
- if (Inserted)<br>
- WorkList.push_back(std::make_pair(SrcReg, SrcVNI));<br>
- propagateSiblingValue(SVI, VNI);<br>
- // Next work list item.<br>
- continue;<br>
- }<br>
- }<br>
-<br>
- // Track reachable reloads.<br>
- SVI->second.DefMI = MI;<br>
- SVI->second.SpillMBB = MI->getParent();<br>
- int FI;<br>
- if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) {<br>
- DEBUG(dbgs() << "reload\n");<br>
- propagateSiblingValue(SVI);<br>
- // Next work list item.<br>
- continue;<br>
- }<br>
-<br>
- // Potential remat candidate.<br>
- DEBUG(dbgs() << "def " << *MI);<br>
- SVI->second.AllDefsAreReloads = false;<br>
- propagateSiblingValue(SVI);<br>
- } while (!WorkList.empty());<br>
-<br>
- // Look up the value we were looking for. We already did this lookup at the<br>
- // top of the function, but SibValues may have been invalidated.<br>
- SVI = SibValues.find(UseVNI);<br>
- assert(SVI != SibValues.end() && "Didn't compute requested info");<br>
- DEBUG(dbgs() << " traced to:\t" << SVI->second);<br>
- return SVI->second.DefMI;<br>
-}<br>
-<br>
-/// analyzeSiblingValues - Trace values defined by sibling copies back to<br>
-/// something that isn't a sibling copy.<br>
+/// This hoist only helps when the copy kills its source.<br>
///<br>
-/// Keep track of values that may be rematerializable.<br>
-void InlineSpiller::analyzeSiblingValues() {<br>
- SibValues.clear();<br>
-<br>
- // No siblings at all?<br>
- if (Edit->getReg() == Original)<br>
- return;<br>
-<br>
- LiveInterval &OrigLI = LIS.getInterval(Original);<br>
- for (unsigned Reg : RegsToSpill) {<br>
- LiveInterval &LI = LIS.getInterval(Reg);<br>
- for (LiveInterval::const_vni_iterator VI = LI.vni_begin(),<br>
- VE = LI.vni_end(); VI != VE; ++VI) {<br>
- VNInfo *VNI = *VI;<br>
- if (VNI->isUnused())<br>
- continue;<br>
- MachineInstr *DefMI = nullptr;<br>
- if (!VNI->isPHIDef()) {<br>
- DefMI = LIS.getInstructionFromIndex(VNI->def);<br>
- assert(DefMI && "No defining instruction");<br>
- }<br>
- // Check possible sibling copies.<br>
- if (VNI->isPHIDef() || DefMI->isCopy()) {<br>
- VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def);<br>
- assert(OrigVNI && "Def outside original live range");<br>
- if (OrigVNI->def != VNI->def)<br>
- DefMI = traceSiblingValue(Reg, VNI, OrigVNI);<br>
- }<br>
- if (DefMI && Edit->checkRematerializable(VNI, DefMI, AA)) {<br>
- DEBUG(dbgs() << "Value " << PrintReg(Reg) << ':' << VNI->id << '@'<br>
- << VNI->def << " may remat from " << *DefMI);<br>
- }<br>
- }<br>
- }<br>
-}<br>
-<br>
-/// hoistSpill - Given a sibling copy that defines a value to be spilled, insert<br>
-/// a spill at a better location.<br>
-bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr &CopyMI) {<br>
+bool InlineSpiller::hoistSpillInsideBB(LiveInterval &SpillLI,<br>
+ MachineInstr &CopyMI) {<br>
SlotIndex Idx = LIS.getInstructionIndex(CopyMI);<br>
+#ifndef NDEBUG<br>
VNInfo *VNI = SpillLI.getVNInfoAt(Idx.getRegSlot());<br>
assert(VNI && VNI->def == Idx.getRegSlot() && "Not defined by copy");<br>
- SibValueMap::iterator I = SibValues.find(VNI);<br>
- if (I == SibValues.end())<br>
- return false;<br>
-<br>
- const SibValueInfo &SVI = I->second;<br>
-<br>
- // Let the normal folding code deal with the boring case.<br>
- if (!SVI.AllDefsAreReloads && SVI.SpillVNI == VNI)<br>
- return false;<br>
-<br>
- // SpillReg may have been deleted by remat and DCE.<br>
- if (!LIS.hasInterval(SVI.SpillReg)) {<br>
- DEBUG(dbgs() << "Stale interval: " << PrintReg(SVI.SpillReg) << '\n');<br>
- SibValues.erase(I);<br>
- return false;<br>
- }<br>
+#endif<br>
<br>
- LiveInterval &SibLI = LIS.getInterval(SVI.SpillReg);<br>
- if (!SibLI.containsValue(SVI.SpillVNI)) {<br>
- DEBUG(dbgs() << "Stale value: " << PrintReg(SVI.SpillReg) << '\n');<br>
- SibValues.erase(I);<br>
+ unsigned SrcReg = CopyMI.getOperand(1).getReg();<br>
+ LiveInterval &SrcLI = LIS.getInterval(SrcReg);<br>
+ VNInfo *SrcVNI = SrcLI.getVNInfoAt(Idx);<br>
+ LiveQueryResult SrcQ = SrcLI.Query(Idx);<br>
+ MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(SrcVNI->def);<br>
+ if (DefMBB != CopyMI.getParent() || !SrcQ.isKill())<br>
return false;<br>
- }<br>
<br>
// Conservatively extend the stack slot range to the range of the original<br>
// value. We may be able to do better with stack slot coloring by being more<br>
@@ -719,35 +365,29 @@ bool InlineSpiller::hoistSpill(LiveInter<br>
DEBUG(dbgs() << "\tmerged orig valno " << OrigVNI->id << ": "<br>
<< *StackInt << '\n');<br>
<br>
- // Already spilled everywhere.<br>
- if (SVI.AllDefsAreReloads) {<br>
- DEBUG(dbgs() << "\tno spill needed: " << SVI);<br>
- ++NumOmitReloadSpill;<br>
- return true;<br>
- }<br>
- // We are going to spill SVI.SpillVNI immediately after its def, so clear out<br>
+ // We are going to spill SrcVNI immediately after its def, so clear out<br>
// any later spills of the same value.<br>
- eliminateRedundantSpills(SibLI, SVI.SpillVNI);<br>
+ eliminateRedundantSpills(SrcLI, SrcVNI);<br>
<br>
- MachineBasicBlock *MBB = LIS.getMBBFromIndex(SVI.SpillVNI->def);<br>
+ MachineBasicBlock *MBB = LIS.getMBBFromIndex(SrcVNI->def);<br>
MachineBasicBlock::iterator MII;<br>
- if (SVI.SpillVNI->isPHIDef())<br>
+ if (SrcVNI->isPHIDef())<br>
MII = MBB->SkipPHIsAndLabels(MBB->begin());<br>
else {<br>
- MachineInstr *DefMI = LIS.getInstructionFromIndex(SVI.SpillVNI->def);<br>
+ MachineInstr *DefMI = LIS.getInstructionFromIndex(SrcVNI->def);<br>
assert(DefMI && "Defining instruction disappeared");<br>
MII = DefMI;<br>
++MII;<br>
}<br>
// Insert spill without kill flag immediately after def.<br>
- TII.storeRegToStackSlot(*MBB, MII, SVI.SpillReg, false, StackSlot,<br>
- MRI.getRegClass(SVI.SpillReg), &TRI);<br>
+ TII.storeRegToStackSlot(*MBB, MII, SrcReg, false, StackSlot,<br>
+ MRI.getRegClass(SrcReg), &TRI);<br>
--MII; // Point to store instruction.<br>
LIS.InsertMachineInstrInMaps(*MII);<br>
- DEBUG(dbgs() << "\thoisted: " << SVI.SpillVNI->def << '\t' << *MII);<br>
+ DEBUG(dbgs() << "\thoisted: " << SrcVNI->def << '\t' << *MII);<br>
<br>
+ HSpiller.addToMergeableSpills(&(*MII), StackSlot, Original);<br>
++NumSpills;<br>
- ++NumHoists;<br>
return true;<br>
}<br>
<br>
@@ -805,7 +445,8 @@ void InlineSpiller::eliminateRedundantSp<br>
MI->setDesc(TII.get(TargetOpcode::KILL));<br>
DeadDefs.push_back(MI);<br>
++NumSpillsRemoved;<br>
- --NumSpills;<br>
+ if (HSpiller.rmFromMergeableSpills(MI, StackSlot))<br>
+ --NumSpills;<br>
}<br>
}<br>
} while (!WorkList.empty());<br>
@@ -876,12 +517,12 @@ bool InlineSpiller::reMaterializeFor(Liv<br>
if (SnippetCopies.count(&MI))<br>
return false;<br>
<br>
- // Use an OrigVNI from traceSiblingValue when ParentVNI is a sibling copy.<br>
+ LiveInterval &OrigLI = LIS.getInterval(Original);<br>
+ VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);<br>
LiveRangeEdit::Remat RM(ParentVNI);<br>
- SibValueMap::const_iterator SibI = SibValues.find(ParentVNI);<br>
- if (SibI != SibValues.end())<br>
- RM.OrigMI = SibI->second.DefMI;<br>
- if (!Edit->canRematerializeAt(RM, UseIdx, false)) {<br>
+ RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);<br>
+<br>
+ if (!Edit->canRematerializeAt(RM, OrigVNI, UseIdx, false)) {<br>
markValueUsed(&VirtReg, ParentVNI);<br>
DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI);<br>
return false;<br>
@@ -931,7 +572,6 @@ bool InlineSpiller::reMaterializeFor(Liv<br>
/// reMaterializeAll - Try to rematerialize as many uses as possible,<br>
/// and trim the live ranges after.<br>
void InlineSpiller::reMaterializeAll() {<br>
- // analyzeSiblingValues has already tested all relevant defining instructions.<br>
if (!Edit->anyRematerializable(AA))<br>
return;<br>
<br>
@@ -1017,6 +657,9 @@ bool InlineSpiller::coalesceStackAccess(<br>
if (InstrReg != Reg || FI != StackSlot)<br>
return false;<br>
<br>
+ if (!IsLoad)<br>
+ HSpiller.rmFromMergeableSpills(MI, StackSlot);<br>
+<br>
DEBUG(dbgs() << "Coalescing stack access: " << *MI);<br>
LIS.RemoveMachineInstrFromMaps(*MI);<br>
MI->eraseFromParent();<br>
@@ -1141,6 +784,9 @@ foldMemoryOperand(ArrayRef<std::pair<Mac<br>
LIS.removePhysRegDefAt(Reg, Idx);<br>
}<br>
<br>
+ int FI;<br>
+ if (TII.isStoreToStackSlot(MI, FI) && HSpiller.rmFromMergeableSpills(MI, FI))<br>
+ --NumSpills;<br>
LIS.ReplaceMachineInstrInMaps(*MI, *FoldMI);<br>
MI->eraseFromParent();<br>
<br>
@@ -1166,9 +812,10 @@ foldMemoryOperand(ArrayRef<std::pair<Mac<br>
<br>
if (!WasCopy)<br>
++NumFolded;<br>
- else if (Ops.front().second == 0)<br>
+ else if (Ops.front().second == 0) {<br>
++NumSpills;<br>
- else<br>
+ HSpiller.addToMergeableSpills(FoldMI, StackSlot, Original);<br>
+ } else<br>
++NumReloads;<br>
return true;<br>
}<br>
@@ -1203,6 +850,7 @@ void InlineSpiller::insertSpill(unsigned<br>
DEBUG(dumpMachineInstrRangeWithSlotIndex(std::next(MI), MIS.end(), LIS,<br>
"spill"));<br>
++NumSpills;<br>
+ HSpiller.addToMergeableSpills(std::next(MI), StackSlot, Original);<br>
}<br>
<br>
/// spillAroundUses - insert spill code around each use of Reg.<br>
@@ -1266,8 +914,7 @@ void InlineSpiller::spillAroundUses(unsi<br>
continue;<br>
}<br>
if (RI.Writes) {<br>
- // Hoist the spill of a sib-reg copy.<br>
- if (hoistSpill(OldLI, *MI)) {<br>
+ if (hoistSpillInsideBB(OldLI, *MI)) {<br>
// This COPY is now dead, the value is already in the stack slot.<br>
MI->getOperand(0).setIsDead();<br>
DeadDefs.push_back(MI);<br>
@@ -1380,7 +1027,6 @@ void InlineSpiller::spill(LiveRangeEdit<br>
assert(DeadDefs.empty() && "Previous spill didn't remove dead defs");<br>
<br>
collectRegsToSpill();<br>
- analyzeSiblingValues();<br>
reMaterializeAll();<br>
<br>
// Remat may handle everything.<br>
@@ -1389,3 +1035,393 @@ void InlineSpiller::spill(LiveRangeEdit<br>
<br>
Edit->calculateRegClassAndHint(MF, Loops, MBFI);<br>
}<br>
+<br>
+/// Optimizations after all the reg selections and spills are done.<br>
+///<br>
+void InlineSpiller::postOptimization() {<br>
+ SmallVector<unsigned, 4> NewVRegs;<br>
+ LiveRangeEdit LRE(nullptr, NewVRegs, MF, LIS, &VRM, nullptr);<br>
+ HSpiller.hoistAllSpills(LRE);<br>
+ assert(NewVRegs.size() == 0 &&<br>
+ "No new vregs should be generated in hoistAllSpills");<br>
+}<br>
+<br>
+/// When a spill is inserted, add the spill to MergeableSpills map.<br>
+///<br>
+void HoistSpillHelper::addToMergeableSpills(MachineInstr *Spill, int StackSlot,<br>
+ unsigned Original) {<br>
+ StackSlotToReg[StackSlot] = Original;<br>
+ SlotIndex Idx = LIS.getInstructionIndex(*Spill);<br>
+ VNInfo *OrigVNI = LIS.getInterval(Original).getVNInfoAt(Idx.getRegSlot());<br>
+ std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI);<br>
+ MergeableSpills[MIdx].insert(Spill);<br>
+}<br>
+<br>
+/// When a spill is removed, remove the spill from MergeableSpills map.<br>
+/// Return true if the spill is removed successfully.<br>
+///<br>
+bool HoistSpillHelper::rmFromMergeableSpills(MachineInstr *Spill,<br>
+ int StackSlot) {<br>
+ int Original = StackSlotToReg[StackSlot];<br>
+ if (!Original)<br>
+ return false;<br>
+ SlotIndex Idx = LIS.getInstructionIndex(*Spill);<br>
+ VNInfo *OrigVNI = LIS.getInterval(Original).getVNInfoAt(Idx.getRegSlot());<br>
+ std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI);<br>
+ return MergeableSpills[MIdx].erase(Spill);<br>
+}<br>
+<br>
+/// Check BB to see if it is a possible target BB to place a hoisted spill,<br>
+/// i.e., there should be a living sibling of OrigReg at the insert point.<br>
+///<br>
+bool HoistSpillHelper::isSpillCandBB(unsigned OrigReg, VNInfo &OrigVNI,<br>
+ MachineBasicBlock &BB, unsigned &LiveReg) {<br>
+ SlotIndex Idx;<br>
+ MachineBasicBlock::iterator MI = BB.getFirstTerminator();<br>
+ if (MI != BB.end())<br>
+ Idx = LIS.getInstructionIndex(*MI);<br>
+ else<br>
+ Idx = LIS.getMBBEndIdx(&BB).getPrevSlot();<br>
+ SmallSetVector<unsigned, 16> &Siblings = Virt2SiblingsMap[OrigReg];<br>
+ assert((LIS.getInterval(OrigReg)).getVNInfoAt(Idx) == &OrigVNI &&<br>
+ "Unexpected VNI");<br>
+<br>
+ for (auto const SibReg : Siblings) {<br>
+ LiveInterval &LI = LIS.getInterval(SibReg);<br>
+ VNInfo *VNI = LI.getVNInfoAt(Idx);<br>
+ if (VNI) {<br>
+ LiveReg = SibReg;<br>
+ return true;<br>
+ }<br>
+ }<br>
+ return false;<br>
+}<br>
+<br>
+/// Remove redundent spills in the same BB. Save those redundent spills in<br>
+/// SpillsToRm, and save the spill to keep and its BB in SpillBBToSpill map.<br>
+///<br>
+void HoistSpillHelper::rmRedundantSpills(<br>
+ SmallPtrSet<MachineInstr *, 16> &Spills,<br>
+ SmallVectorImpl<MachineInstr *> &SpillsToRm,<br>
+ DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) {<br>
+ // For each spill saw, check SpillBBToSpill[] and see if its BB already has<br>
+ // another spill inside. If a BB contains more than one spill, only keep the<br>
+ // earlier spill with smaller SlotIndex.<br>
+ for (const auto CurrentSpill : Spills) {<br>
+ MachineBasicBlock *Block = CurrentSpill->getParent();<br>
+ MachineDomTreeNode *Node = MDT.DT->getNode(Block);<br>
+ MachineInstr *PrevSpill = SpillBBToSpill[Node];<br>
+ if (PrevSpill) {<br>
+ SlotIndex PIdx = LIS.getInstructionIndex(*PrevSpill);<br>
+ SlotIndex CIdx = LIS.getInstructionIndex(*CurrentSpill);<br>
+ MachineInstr *SpillToRm = (CIdx > PIdx) ? CurrentSpill : PrevSpill;<br>
+ MachineInstr *SpillToKeep = (CIdx > PIdx) ? PrevSpill : CurrentSpill;<br>
+ SpillsToRm.push_back(SpillToRm);<br>
+ SpillBBToSpill[MDT.DT->getNode(Block)] = SpillToKeep;<br>
+ } else {<br>
+ SpillBBToSpill[MDT.DT->getNode(Block)] = CurrentSpill;<br>
+ }<br>
+ }<br>
+ for (const auto SpillToRm : SpillsToRm)<br>
+ Spills.erase(SpillToRm);<br>
+}<br>
+<br>
+/// Starting from \p Root find a top-down traversal order of the dominator<br>
+/// tree to visit all basic blocks containing the elements of \p Spills.<br>
+/// Redundant spills will be found and put into \p SpillsToRm at the same<br>
+/// time. \p SpillBBToSpill will be populated as part of the process and<br>
+/// maps a basic block to the first store occurring in the basic block.<br>
+/// \post SpillsToRm.union(Spills@post) == Spills@pre<br>
+///<br>
+void HoistSpillHelper::getVisitOrders(<br>
+ MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills,<br>
+ SmallVectorImpl<MachineDomTreeNode *> &Orders,<br>
+ SmallVectorImpl<MachineInstr *> &SpillsToRm,<br>
+ DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep,<br>
+ DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) {<br>
+ // The set contains all the possible BB nodes to which we may hoist<br>
+ // original spills.<br>
+ SmallPtrSet<MachineDomTreeNode *, 8> WorkSet;<br>
+ // Save the BB nodes on the path from the first BB node containing<br>
+ // non-redundent spill to the Root node.<br>
+ SmallPtrSet<MachineDomTreeNode *, 8> NodesOnPath;<br>
+ // All the spills to be hoisted must originate from a single def instruction<br>
+ // to the OrigReg. It means the def instruction should dominate all the spills<br>
+ // to be hoisted. We choose the BB where the def instruction is located as<br>
+ // the Root.<br>
+ MachineDomTreeNode *RootIDomNode = MDT[Root]->getIDom();<br>
+ // For every node on the dominator tree with spill, walk up on the dominator<br>
+ // tree towards the Root node until it is reached. If there is other node<br>
+ // containing spill in the middle of the path, the previous spill saw will<br>
+ // be redundent and the node containing it will be removed. All the nodes on<br>
+ // the path starting from the first node with non-redundent spill to the Root<br>
+ // node will be added to the WorkSet, which will contain all the possible<br>
+ // locations where spills may be hoisted to after the loop below is done.<br>
+ for (const auto Spill : Spills) {<br>
+ MachineBasicBlock *Block = Spill->getParent();<br>
+ MachineDomTreeNode *Node = MDT[Block];<br>
+ MachineInstr *SpillToRm = nullptr;<br>
+ while (Node != RootIDomNode) {<br>
+ // If Node dominates Block, and it already contains a spill, the spill in<br>
+ // Block will be redundent.<br>
+ if (Node != MDT[Block] && SpillBBToSpill[Node]) {<br>
+ SpillToRm = SpillBBToSpill[MDT[Block]];<br>
+ break;<br>
+ /// If we see the Node already in WorkSet, the path from the Node to<br>
+ /// the Root node must already be traversed by another spill.<br>
+ /// Then no need to repeat.<br>
+ } else if (WorkSet.count(Node)) {<br>
+ break;<br>
+ } else {<br>
+ NodesOnPath.insert(Node);<br>
+ }<br>
+ Node = Node->getIDom();<br>
+ }<br>
+ if (SpillToRm) {<br>
+ SpillsToRm.push_back(SpillToRm);<br>
+ } else {<br>
+ // Add a BB containing the original spills to SpillsToKeep -- i.e.,<br>
+ // set the initial status before hoisting start. The value of BBs<br>
+ // containing original spills is set to 0, in order to descriminate<br>
+ // with BBs containing hoisted spills which will be inserted to<br>
+ // SpillsToKeep later during hoisting.<br>
+ SpillsToKeep[MDT[Block]] = 0;<br>
+ WorkSet.insert(NodesOnPath.begin(), NodesOnPath.end());<br>
+ }<br>
+ NodesOnPath.clear();<br>
+ }<br>
+<br>
+ // Sort the nodes in WorkSet in top-down order and save the nodes<br>
+ // in Orders. Orders will be used for hoisting in runHoistSpills.<br>
+ unsigned idx = 0;<br>
+ Orders.push_back(MDT.DT->getNode(Root));<br>
+ do {<br>
+ MachineDomTreeNode *Node = Orders[idx++];<br>
+ const std::vector<MachineDomTreeNode *> &Children = Node->getChildren();<br>
+ unsigned NumChildren = Children.size();<br>
+ for (unsigned i = 0; i != NumChildren; ++i) {<br>
+ MachineDomTreeNode *Child = Children[i];<br>
+ if (WorkSet.count(Child))<br>
+ Orders.push_back(Child);<br>
+ }<br>
+ } while (idx != Orders.size());<br>
+ assert(Orders.size() == WorkSet.size() &&<br>
+ "Orders have different size with WorkSet");<br>
+<br>
+#ifndef NDEBUG<br>
+ DEBUG(dbgs() << "Orders size is " << Orders.size() << "\n");<br>
+ SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin();<br>
+ for (; RIt != Orders.rend(); RIt++)<br>
+ DEBUG(dbgs() << "BB" << (*RIt)->getBlock()->getNumber() << ",");<br>
+ DEBUG(dbgs() << "\n");<br>
+#endif<br>
+}<br>
+<br>
+/// Try to hoist spills according to BB hotness. The spills to removed will<br>
+/// be saved in \p SpillsToRm. The spills to be inserted will be saved in<br>
+/// \p SpillsToIns.<br>
+///<br>
+void HoistSpillHelper::runHoistSpills(<br>
+ unsigned OrigReg, VNInfo &OrigVNI, SmallPtrSet<MachineInstr *, 16> &Spills,<br>
+ SmallVectorImpl<MachineInstr *> &SpillsToRm,<br>
+ DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns) {<br>
+ // Visit order of dominator tree nodes.<br>
+ SmallVector<MachineDomTreeNode *, 32> Orders;<br>
+ // SpillsToKeep contains all the nodes where spills are to be inserted<br>
+ // during hoisting. If the spill to be inserted is an original spill<br>
+ // (not a hoisted one), the value of the map entry is 0. If the spill<br>
+ // is a hoisted spill, the value of the map entry is the VReg to be used<br>
+ // as the source of the spill.<br>
+ DenseMap<MachineDomTreeNode *, unsigned> SpillsToKeep;<br>
+ // Map from BB to the first spill inside of it.<br>
+ DenseMap<MachineDomTreeNode *, MachineInstr *> SpillBBToSpill;<br>
+<br>
+ rmRedundantSpills(Spills, SpillsToRm, SpillBBToSpill);<br>
+<br>
+ MachineBasicBlock *Root = LIS.getMBBFromIndex(OrigVNI.def);<br>
+ getVisitOrders(Root, Spills, Orders, SpillsToRm, SpillsToKeep,<br>
+ SpillBBToSpill);<br>
+<br>
+ // SpillsInSubTreeMap keeps the map from a dom tree node to a pair of<br>
+ // nodes set and the cost of all the spills inside those nodes.<br>
+ // The nodes set are the locations where spills are to be inserted<br>
+ // in the subtree of current node.<br>
+ typedef std::pair<SmallPtrSet<MachineDomTreeNode *, 16>, BlockFrequency><br>
+ NodesCostPair;<br>
+ DenseMap<MachineDomTreeNode *, NodesCostPair> SpillsInSubTreeMap;<br>
+ // Iterate Orders set in reverse order, which will be a bottom-up order<br>
+ // in the dominator tree. Once we visit a dom tree node, we know its<br>
+ // children have already been visited and the spill locations in the<br>
+ // subtrees of all the children have been determined.<br>
+ SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin();<br>
+ for (; RIt != Orders.rend(); RIt++) {<br>
+ MachineBasicBlock *Block = (*RIt)->getBlock();<br>
+<br>
+ // If Block contains an original spill, simply continue.<br>
+ if (SpillsToKeep.find(*RIt) != SpillsToKeep.end() && !SpillsToKeep[*RIt]) {<br>
+ SpillsInSubTreeMap[*RIt].first.insert(*RIt);<br>
+ // SpillsInSubTreeMap[*RIt].second contains the cost of spill.<br>
+ SpillsInSubTreeMap[*RIt].second = MBFI.getBlockFreq(Block);<br>
+ continue;<br>
+ }<br>
+<br>
+ // Collect spills in subtree of current node (*RIt) to<br>
+ // SpillsInSubTreeMap[*RIt].first.<br>
+ const std::vector<MachineDomTreeNode *> &Children = (*RIt)->getChildren();<br>
+ unsigned NumChildren = Children.size();<br>
+ for (unsigned i = 0; i != NumChildren; ++i) {<br>
+ MachineDomTreeNode *Child = Children[i];<br>
+ auto BI = SpillsInSubTreeMap[Child].first.begin();<br>
+ auto EI = SpillsInSubTreeMap[Child].first.end();<br>
+ SpillsInSubTreeMap[*RIt].first.insert(BI, EI);<br>
+ SpillsInSubTreeMap[*RIt].second += SpillsInSubTreeMap[Child].second;<br>
+ SpillsInSubTreeMap.erase(Child);<br>
+ }<br>
+<br>
+ // No spills in subtree, simply continue.<br>
+ if (SpillsInSubTreeMap[*RIt].first.empty())<br>
+ continue;<br>
+<br>
+ // Check whether Block is a possible candidate to insert spill.<br>
+ unsigned LiveReg = 0;<br>
+ if (!isSpillCandBB(OrigReg, OrigVNI, *Block, LiveReg))<br>
+ continue;<br>
+<br>
+ // If there are multiple spills that could be merged, bias a little<br>
+ // to hoist the spill.<br>
+ BranchProbability MarginProb = (SpillsInSubTreeMap[*RIt].first.size() > 1)<br>
+ ? BranchProbability(9, 10)<br>
+ : BranchProbability(1, 1);<br>
+ if (SpillsInSubTreeMap[*RIt].second ><br>
+ MBFI.getBlockFreq(Block) * MarginProb) {<br>
+ // Hoist: Move spills to current Block.<br>
+ for (const auto SpillBB : SpillsInSubTreeMap[*RIt].first) {<br>
+ // When SpillBB is a BB contains original spill, insert the spill<br>
+ // to SpillsToRm.<br>
+ if (SpillsToKeep.find(SpillBB) != SpillsToKeep.end() &&<br>
+ !SpillsToKeep[SpillBB]) {<br>
+ MachineInstr *SpillToRm = SpillBBToSpill[SpillBB];<br>
+ SpillsToRm.push_back(SpillToRm);<br>
+ }<br>
+ // SpillBB will not contain spill anymore, remove it from SpillsToKeep.<br>
+ SpillsToKeep.erase(SpillBB);<br>
+ }<br>
+ // Current Block is the BB containing the new hoisted spill. Add it to<br>
+ // SpillsToKeep. LiveReg is the source of the new spill.<br>
+ SpillsToKeep[*RIt] = LiveReg;<br>
+ DEBUG({<br>
+ dbgs() << "spills in BB: ";<br>
+ for (const auto Rspill : SpillsInSubTreeMap[*RIt].first)<br>
+ dbgs() << Rspill->getBlock()->getNumber() << " ";<br>
+ dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber()<br>
+ << "\n";<br>
+ });<br>
+ SpillsInSubTreeMap[*RIt].first.clear();<br>
+ SpillsInSubTreeMap[*RIt].first.insert(*RIt);<br>
+ SpillsInSubTreeMap[*RIt].second = MBFI.getBlockFreq(Block);<br>
+ }<br>
+ }<br>
+ // For spills in SpillsToKeep with LiveReg set (i.e., not original spill),<br>
+ // save them to SpillsToIns.<br>
+ for (const auto Ent : SpillsToKeep) {<br>
+ if (Ent.second)<br>
+ SpillsToIns[Ent.first->getBlock()] = Ent.second;<br>
+ }<br>
+}<br>
+<br>
+/// For spills with equal values, remove redundent spills and hoist the left<br>
+/// to less hot spots.<br>
+///<br>
+/// Spills with equal values will be collected into the same set in<br>
+/// MergeableSpills when spill is inserted. These equal spills are originated<br>
+/// from the same define instruction and are dominated by the instruction.<br>
+/// Before hoisting all the equal spills, redundent spills inside in the same<br>
+/// BB is first marked to be deleted. Then starting from spills left, walk up<br>
+/// on the dominator tree towards the Root node where the define instruction<br>
+/// is located, mark the dominated spills to be deleted along the way and<br>
+/// collect the BB nodes on the path from non-dominated spills to the define<br>
+/// instruction into a WorkSet. The nodes in WorkSet are the candidate places<br>
+/// where we consider to hoist the spills. We iterate the WorkSet in bottom-up<br>
+/// order, and for each node, we will decide whether to hoist spills inside<br>
+/// its subtree to that node. In this way, we can get benefit locally even if<br>
+/// hoisting all the equal spills to one cold place is impossible.<br>
+///<br>
+void HoistSpillHelper::hoistAllSpills(LiveRangeEdit &Edit) {<br>
+ // Save the mapping between stackslot and its original reg.<br>
+ DenseMap<int, unsigned> SlotToOrigReg;<br>
+ for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) {<br>
+ unsigned Reg = TargetRegisterInfo::index2VirtReg(i);<br>
+ int Slot = VRM.getStackSlot(Reg);<br>
+ if (Slot != VirtRegMap::NO_STACK_SLOT)<br>
+ SlotToOrigReg[Slot] = VRM.getOriginal(Reg);<br>
+ unsigned Original = VRM.getPreSplitReg(Reg);<br>
+ if (!MRI.def_empty(Reg))<br>
+ Virt2SiblingsMap[Original].insert(Reg);<br>
+ }<br>
+<br>
+ // Each entry in MergeableSpills contains a spill set with equal values.<br>
+ for (auto &Ent : MergeableSpills) {<br>
+ int Slot = Ent.first.first;<br>
+ unsigned OrigReg = SlotToOrigReg[Slot];<br>
+ VNInfo *OrigVNI = Ent.first.second;<br>
+ SmallPtrSet<MachineInstr *, 16> &EqValSpills = Ent.second;<br>
+ if (Ent.second.empty())<br>
+ continue;<br>
+<br>
+ DEBUG({<br>
+ dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n"<br>
+ << "Equal spills in BB: ";<br>
+ for (const auto spill : EqValSpills)<br>
+ dbgs() << spill->getParent()->getNumber() << " ";<br>
+ dbgs() << "\n";<br>
+ });<br>
+<br>
+ // SpillsToRm is the spill set to be removed from EqValSpills.<br>
+ SmallVector<MachineInstr *, 16> SpillsToRm;<br>
+ // SpillsToIns is the spill set to be newly inserted after hoisting.<br>
+ DenseMap<MachineBasicBlock *, unsigned> SpillsToIns;<br>
+<br>
+ runHoistSpills(OrigReg, *OrigVNI, EqValSpills, SpillsToRm, SpillsToIns);<br>
+<br>
+ DEBUG({<br>
+ dbgs() << "Finally inserted spills in BB: ";<br>
+ for (const auto Ispill : SpillsToIns)<br>
+ dbgs() << Ispill.first->getNumber() << " ";<br>
+ dbgs() << "\nFinally removed spills in BB: ";<br>
+ for (const auto Rspill : SpillsToRm)<br>
+ dbgs() << Rspill->getParent()->getNumber() << " ";<br>
+ dbgs() << "\n";<br>
+ });<br>
+<br>
+ // Stack live range update.<br>
+ LiveInterval &StackIntvl = LSS.getInterval(Slot);<br>
+ if (!SpillsToIns.empty() || !SpillsToRm.empty()) {<br>
+ LiveInterval &OrigLI = LIS.getInterval(OrigReg);<br>
+ StackIntvl.MergeValueInAsValue(OrigLI, OrigVNI,<br>
+ StackIntvl.getValNumInfo(0));<br>
+ }<br>
+<br>
+ // Insert hoisted spills.<br>
+ for (auto const Insert : SpillsToIns) {<br>
+ MachineBasicBlock *BB = Insert.first;<br>
+ unsigned LiveReg = Insert.second;<br>
+ MachineBasicBlock::iterator MI = BB->getFirstTerminator();<br>
+ TII.storeRegToStackSlot(*BB, MI, LiveReg, false, Slot,<br>
+ MRI.getRegClass(LiveReg), &TRI);<br>
+ LIS.InsertMachineInstrRangeInMaps(std::prev(MI), MI);<br>
+ ++NumSpills;<br>
+ }<br>
+<br>
+ // Remove redundent spills or change them to dead instructions.<br>
+ NumSpills -= SpillsToRm.size();<br>
+ for (auto const RMEnt : SpillsToRm) {<br>
+ RMEnt->setDesc(TII.get(TargetOpcode::KILL));<br>
+ for (unsigned i = RMEnt->getNumOperands(); i; --i) {<br>
+ MachineOperand &MO = RMEnt->getOperand(i - 1);<br>
+ if (MO.isReg() && MO.isImplicit() && MO.isDef() && !MO.isDead())<br>
+ RMEnt->RemoveOperand(i - 1);<br>
+ }<br>
+ }<br>
+ Edit.eliminateDeadDefs(SpillsToRm, None, true);<br>
+ }<br>
+}<br>
<br>
Modified: llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp Wed Apr 6 10:41:07 2016<br>
@@ -63,10 +63,13 @@ void LiveRangeEdit::scanRemattable(Alias<br>
for (VNInfo *VNI : getParent().valnos) {<br>
if (VNI->isUnused())<br>
continue;<br>
- MachineInstr *DefMI = LIS.getInstructionFromIndex(VNI->def);<br>
+ unsigned Original = VRM->getOriginal(getReg());<br>
+ LiveInterval &OrigLI = LIS.getInterval(Original);<br>
+ VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def);<br>
+ MachineInstr *DefMI = LIS.getInstructionFromIndex(OrigVNI->def);<br>
if (!DefMI)<br>
continue;<br>
- checkRematerializable(VNI, DefMI, aa);<br>
+ checkRematerializable(OrigVNI, DefMI, aa);<br>
}<br>
ScannedRemattable = true;<br>
}<br>
@@ -113,24 +116,18 @@ bool LiveRangeEdit::allUsesAvailableAt(c<br>
return true;<br>
}<br>
<br>
-bool LiveRangeEdit::canRematerializeAt(Remat &RM,<br>
- SlotIndex UseIdx,<br>
- bool cheapAsAMove) {<br>
+bool LiveRangeEdit::canRematerializeAt(Remat &RM, VNInfo *OrigVNI,<br>
+ SlotIndex UseIdx, bool cheapAsAMove) {<br>
assert(ScannedRemattable && "Call anyRematerializable first");<br>
<br>
// Use scanRemattable info.<br>
- if (!Remattable.count(RM.ParentVNI))<br>
+ if (!Remattable.count(OrigVNI))<br>
return false;<br>
<br>
// No defining instruction provided.<br>
SlotIndex DefIdx;<br>
- if (RM.OrigMI)<br>
- DefIdx = LIS.getInstructionIndex(*RM.OrigMI);<br>
- else {<br>
- DefIdx = RM.ParentVNI->def;<br>
- RM.OrigMI = LIS.getInstructionFromIndex(DefIdx);<br>
- assert(RM.OrigMI && "No defining instruction for remattable value");<br>
- }<br>
+ assert(RM.OrigMI && "No defining instruction for remattable value");<br>
+ DefIdx = LIS.getInstructionIndex(*RM.OrigMI);<br>
<br>
// If only cheap remats were requested, bail out early.<br>
if (cheapAsAMove && !TII.isAsCheapAsAMove(RM.OrigMI))<br>
@@ -261,6 +258,15 @@ void LiveRangeEdit::eliminateDeadDef(Mac<br>
// Collect virtual registers to be erased after MI is gone.<br>
SmallVector<unsigned, 8> RegsToErase;<br>
bool ReadsPhysRegs = false;<br>
+ bool isOrigDef = false;<br>
+ unsigned Dest;<br>
+ if (VRM && MI->getOperand(0).isReg()) {<br>
+ Dest = MI->getOperand(0).getReg();<br>
+ unsigned Original = VRM->getOriginal(Dest);<br>
+ LiveInterval &OrigLI = LIS.getInterval(Original);<br>
+ VNInfo *OrigVNI = OrigLI.getVNInfoAt(Idx);<br>
+ isOrigDef = SlotIndex::isSameInstr(OrigVNI->def, Idx);<br>
+ }<br>
<br>
// Check for live intervals that may shrink<br>
for (MachineInstr::mop_iterator MOI = MI->operands_begin(),<br>
@@ -314,11 +320,24 @@ void LiveRangeEdit::eliminateDeadDef(Mac<br>
}<br>
DEBUG(dbgs() << "Converted physregs to:\t" << *MI);<br>
} else {<br>
- if (TheDelegate)<br>
- TheDelegate->LRE_WillEraseInstruction(MI);<br>
- LIS.RemoveMachineInstrFromMaps(*MI);<br>
- MI->eraseFromParent();<br>
- ++NumDCEDeleted;<br>
+ // If the dest of MI is an original reg, don't delete the inst. Replace<br>
+ // the dest with a new reg, keep the inst for remat of other siblings.<br>
+ // The inst is saved in LiveRangeEdit::DeadRemats and will be deleted<br>
+ // after all the allocations of the func are done.<br>
+ if (isOrigDef) {<br>
+ unsigned NewDest = createFrom(Dest);<br>
+ pop_back();<br>
+ markDeadRemat(MI);<br>
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();<br>
+ MI->substituteRegister(Dest, NewDest, 0, TRI);<br>
+ MI->getOperand(0).setIsDead(false);<br>
+ } else {<br>
+ if (TheDelegate)<br>
+ TheDelegate->LRE_WillEraseInstruction(MI);<br>
+ LIS.RemoveMachineInstrFromMaps(*MI);<br>
+ MI->eraseFromParent();<br>
+ ++NumDCEDeleted;<br>
+ }<br>
}<br>
<br>
// Erase any virtregs that are now empty and unused. There may be <undef><br>
@@ -332,8 +351,9 @@ void LiveRangeEdit::eliminateDeadDef(Mac<br>
}<br>
}<br>
<br>
-void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,<br>
- ArrayRef<unsigned> RegsBeingSpilled) {<br>
+void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr *> &Dead,<br>
+ ArrayRef<unsigned> RegsBeingSpilled,<br>
+ bool NoSplit) {<br>
ToShrinkSet ToShrink;<br>
<br>
for (;;) {<br>
@@ -355,6 +375,9 @@ void LiveRangeEdit::eliminateDeadDefs(Sm<br>
if (!LIS.shrinkToUses(LI, &Dead))<br>
continue;<br>
<br>
+ if (NoSplit)<br>
+ continue;<br>
+<br>
// Don't create new intervals for a register being spilled.<br>
// The new intervals would have to be spilled anyway so its not worth it.<br>
// Also they currently aren't spilled so creating them and not spilling<br>
<br>
Modified: llvm/trunk/lib/CodeGen/RegAllocBase.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBase.cpp?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBase.cpp?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/RegAllocBase.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/RegAllocBase.cpp Wed Apr 6 10:41:07 2016<br>
@@ -153,3 +153,12 @@ void RegAllocBase::allocatePhysRegs() {<br>
}<br>
}<br>
}<br>
+<br>
+void RegAllocBase::postOptimization() {<br>
+ spiller().postOptimization();<br>
+ for (auto DeadInst : DeadRemats) {<br>
+ LIS->RemoveMachineInstrFromMaps(*DeadInst);<br>
+ DeadInst->eraseFromParent();<br>
+ }<br>
+ DeadRemats.clear();<br>
+}<br>
<br>
Modified: llvm/trunk/lib/CodeGen/RegAllocBase.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBase.h?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBase.h?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/RegAllocBase.h (original)<br>
+++ llvm/trunk/lib/CodeGen/RegAllocBase.h Wed Apr 6 10:41:07 2016<br>
@@ -65,6 +65,12 @@ protected:<br>
LiveRegMatrix *Matrix;<br>
RegisterClassInfo RegClassInfo;<br>
<br>
+ /// Inst which is a def of an original reg and whose defs are already all<br>
+ /// dead after remat is saved in DeadRemats. The deletion of such inst is<br>
+ /// postponed till all the allocations are done, so its remat expr is<br>
+ /// always available for the remat of all the siblings of the original reg.<br>
+ SmallPtrSet<MachineInstr *, 32> DeadRemats;<br>
+<br>
RegAllocBase()<br>
: TRI(nullptr), MRI(nullptr), VRM(nullptr), LIS(nullptr), Matrix(nullptr) {}<br>
<br>
@@ -77,6 +83,10 @@ protected:<br>
// physical register assignments.<br>
void allocatePhysRegs();<br>
<br>
+ // Include spiller post optimization and removing dead defs left because of<br>
+ // rematerialization.<br>
+ virtual void postOptimization();<br>
+<br>
// Get a temporary reference to a Spiller instance.<br>
virtual Spiller &spiller() = 0;<br>
<br>
<br>
Modified: llvm/trunk/lib/CodeGen/RegAllocBasic.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBasic.cpp?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBasic.cpp?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/RegAllocBasic.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/RegAllocBasic.cpp Wed Apr 6 10:41:07 2016<br>
@@ -199,7 +199,7 @@ bool RABasic::spillInterferences(LiveInt<br>
Matrix->unassign(Spill);<br>
<br>
// Spill the extracted interval.<br>
- LiveRangeEdit LRE(&Spill, SplitVRegs, *MF, *LIS, VRM);<br>
+ LiveRangeEdit LRE(&Spill, SplitVRegs, *MF, *LIS, VRM, nullptr, &DeadRemats);<br>
spiller().spill(LRE);<br>
}<br>
return true;<br>
@@ -258,7 +258,7 @@ unsigned RABasic::selectOrSplit(LiveInte<br>
DEBUG(dbgs() << "spilling: " << VirtReg << '\n');<br>
if (!VirtReg.isSpillable())<br>
return ~0u;<br>
- LiveRangeEdit LRE(&VirtReg, SplitVRegs, *MF, *LIS, VRM);<br>
+ LiveRangeEdit LRE(&VirtReg, SplitVRegs, *MF, *LIS, VRM, nullptr, &DeadRemats);<br>
spiller().spill(LRE);<br>
<br>
// The live virtual register requesting allocation was spilled, so tell<br>
@@ -283,6 +283,7 @@ bool RABasic::runOnMachineFunction(Machi<br>
SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM));<br>
<br>
allocatePhysRegs();<br>
+ postOptimization();<br>
<br>
// Diagnostic output before rewriting<br>
DEBUG(dbgs() << "Post alloc VirtRegMap:\n" << *VRM << "\n");<br>
<br>
Modified: llvm/trunk/lib/CodeGen/RegAllocGreedy.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocGreedy.cpp?rev=265547&r1=265546&r2=265547&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocGreedy.cpp?rev=265547&r1=265546&r2=265547&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/RegAllocGreedy.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/RegAllocGreedy.cpp Wed Apr 6 10:41:07 2016<br>
@@ -12,7 +12,6 @@<br>
//<br>
//===----------------------------------------------------------------------===//<br>
<br>
-#include "llvm/CodeGen/Passes.h"<br>
#include "AllocationOrder.h"<br>
#include "InterferenceCache.h"<br>
#include "LiveDebugVariables.h"<br>
@@ -33,6 +32,7 @@<br>
#include "llvm/CodeGen/MachineFunctionPass.h"<br>
#include "llvm/CodeGen/MachineLoopInfo.h"<br>
#include "llvm/CodeGen/MachineRegisterInfo.h"<br>
+#include "llvm/CodeGen/Passes.h"<br>
#include "llvm/CodeGen/RegAllocRegistry.h"<br>
#include "llvm/CodeGen/RegisterClassInfo.h"<br>
#include "llvm/CodeGen/VirtRegMap.h"<br>
@@ -44,6 +44,7 @@<br>
#include "llvm/Support/ErrorHandling.h"<br>
#include "llvm/Support/Timer.h"<br>
#include "llvm/Support/raw_ostream.h"<br>
+#include "llvm/Target/TargetInstrInfo.h"<br>
#include "llvm/Target/TargetSubtargetInfo.h"<br>
#include <queue><br>
<br>
@@ -55,14 +56,14 @@ STATISTIC(NumGlobalSplits, "Number of sp<br>
STATISTIC(NumLocalSplits, "Number of split local live ranges");<br>
STATISTIC(NumEvicted, "Number of interferences evicted");<br>
<br>
-static cl::opt<SplitEditor::ComplementSpillMode><br>
-SplitSpillMode("split-spill-mode", cl::Hidden,<br>
- cl::desc("Spill mode for splitting live ranges"),<br>
- cl::values(clEnumValN(SplitEditor::SM_Partition, "default", "Default"),<br>
- clEnumValN(SplitEditor::SM_Size, "size", "Optimize for size"),<br>
- clEnumValN(SplitEditor::SM_Speed, "speed", "Optimize for speed"),<br>
- clEnumValEnd),<br>
- cl::init(SplitEditor::SM_Partition));<br>
+static cl::opt<SplitEditor::ComplementSpillMode> SplitSpillMode(<br>
+ "split-spill-mode", cl::Hidden,<br>
+ cl::desc("Spill mode for splitting live ranges"),<br>
+ cl::values(clEnumValN(SplitEditor::SM_Partition, "default", "Default"),<br>
+ clEnumValN(SplitEditor::SM_Size, "size", "Optimize for size"),<br>
+ clEnumValN(SplitEditor::SM_Speed, "speed", "Optimize for speed"),<br>
+ clEnumValEnd),<br>
+ cl::init(SplitEditor::SM_Speed));<br>
<br>
static cl::opt<unsigned><br>
LastChanceRecoloringMaxDepth("lcr-max-depth", cl::Hidden,<br>
@@ -1465,7 +1466,7 @@ unsigned RAGreedy::doRegionSplit(LiveInt<br>
SmallVectorImpl<unsigned> &NewVRegs) {<br>
SmallVector<unsigned, 8> UsedCands;<br>
// Prepare split editor.<br>
- LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);<br>
+ LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);<br>
SE->reset(LREdit, SplitSpillMode);<br>
<br>
// Assign all edge bundles to the preferred candidate, or NoCand.<br>
@@ -1513,7 +1514,7 @@ unsigned RAGreedy::tryBlockSplit(LiveInt<br>
assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed");<br>
unsigned Reg = VirtReg.reg;<br>
bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg));<br>
- LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);<br>
+ LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);<br>
SE->reset(LREdit, SplitSpillMode);<br>
ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();<br>
for (unsigned i = 0; i != UseBlocks.size(); ++i) {<br>
@@ -1585,7 +1586,7 @@ RAGreedy::tryInstructionSplit(LiveInterv<br>
<br>
// Always enable split spill mode, since we're effectively spilling to a<br>
// register.<br>
- LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);<br>
+ LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);<br>
SE->reset(LREdit, SplitEditor::SM_Size);<br>
<br>
ArrayRef<SlotIndex> Uses = SA->getUseSlots();<br>
@@ -1908,7 +1909,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInt<br>
<< '-' << Uses[BestAfter] << ", " << BestDiff<br>
<< ", " << (BestAfter - BestBefore + 1) << " instrs\n");<br>
<br>
- LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);<br>
+ LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);<br>
SE->reset(LREdit);<br>
<br>
SE->openIntv();<br>
@@ -2551,7 +2552,7 @@ unsigned RAGreedy::selectOrSplitImpl(Liv<br>
NewVRegs.push_back(VirtReg.reg);<br>
} else {<br>
NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled);<br>
- LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this);<br>
+ LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);<br>
spiller().spill(LRE);<br>
setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);<br>
<br>
@@ -2609,6 +2610,8 @@ bool RAGreedy::runOnMachineFunction(Mach<br>
<br>
allocatePhysRegs();<br>
tryHintsRecoloring();<br>
+ postOptimization();<br>
+<br>
</blockquote></div><div dir="ltr">-- <br></div>Mike<br>Sent from phone