[llvm] r279625 - Create subranges for new intervals resulting from live interval splitting
Krzysztof Parzyszek via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 24 06:37:56 PDT 2016
Author: kparzysz
Date: Wed Aug 24 08:37:55 2016
New Revision: 279625
URL: http://llvm.org/viewvc/llvm-project?rev=279625&view=rev
Log:
Create subranges for new intervals resulting from live interval splitting
The register allocator can split a live interval of a register into a set
of smaller intervals. After the allocation of registers is complete, the
rewriter will modify the IR to replace virtual registers with the corres-
ponding physical registers. At this stage, if a register corresponding
to a subregister of a virtual register is used, the rewriter will check
if that subregister is undefined, and if so, it will add the <undef> flag
to the machine operand. The function verifying liveness of the subregis-
ter would assume that it is undefined, unless any of the subranges of the
live interval proves otherwise.
The problem is that the live intervals created during splitting do not
have any subranges, even if the original parent interval did. This could
result in the <undef> flag placed on a register that is actually defined.
Differential Revision: http://reviews.llvm.org/D21189
Added:
llvm/trunk/test/CodeGen/AMDGPU/subreg-intervals.mir
llvm/trunk/test/CodeGen/Hexagon/regalloc-bad-undef.mir
Modified:
llvm/trunk/include/llvm/CodeGen/LiveInterval.h
llvm/trunk/include/llvm/CodeGen/LiveIntervalAnalysis.h
llvm/trunk/lib/CodeGen/LiveInterval.cpp
llvm/trunk/lib/CodeGen/LiveIntervalAnalysis.cpp
llvm/trunk/lib/CodeGen/LiveRangeCalc.cpp
llvm/trunk/lib/CodeGen/LiveRangeCalc.h
llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp
llvm/trunk/lib/CodeGen/MachineVerifier.cpp
llvm/trunk/lib/CodeGen/RegAllocBase.cpp
llvm/trunk/lib/CodeGen/RegisterCoalescer.cpp
llvm/trunk/lib/CodeGen/RenameIndependentSubregs.cpp
llvm/trunk/lib/CodeGen/SplitKit.cpp
llvm/trunk/lib/CodeGen/SplitKit.h
llvm/trunk/lib/CodeGen/VirtRegMap.cpp
llvm/trunk/lib/Target/Hexagon/HexagonExpandCondsets.cpp
Modified: llvm/trunk/include/llvm/CodeGen/LiveInterval.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/LiveInterval.h?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/LiveInterval.h (original)
+++ llvm/trunk/include/llvm/CodeGen/LiveInterval.h Wed Aug 24 08:37:55 2016
@@ -316,6 +316,10 @@ namespace llvm {
/// add liveness for a dead def.
VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator &VNInfoAllocator);
+ /// Create a def of value @p VNI. Return @p VNI. If there already exists
+ /// a definition at VNI->def, the value defined there must be @p VNI.
+ VNInfo *createDeadDef(VNInfo *VNI);
+
/// Create a copy of the given value. The new value will be identical except
/// for the Value number.
VNInfo *createValueCopy(const VNInfo *orig,
@@ -451,10 +455,29 @@ namespace llvm {
/// may have grown since it was inserted).
iterator addSegment(Segment S);
+ /// Attempt to extend a value defined after @p StartIdx to include @p Use.
+ /// Both @p StartIdx and @p Use should be in the same basic block. In case
+ /// of subranges, an extension could be prevented by an explicit "undef"
+ /// caused by a <def,read-undef> on a non-overlapping lane. The list of
+ /// location of such "undefs" should be provided in @p Undefs.
+ /// The return value is a pair: the first element is VNInfo of the value
+ /// that was extended (possibly nullptr), the second is a boolean value
+ /// indicating whether an "undef" was encountered.
/// If this range is live before @p Use in the basic block that starts at
- /// @p StartIdx, extend it to be live up to @p Use, and return the value. If
- /// there is no segment before @p Use, return nullptr.
- VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Use);
+ /// @p StartIdx, and there is no intervening "undef", extend it to be live
+ /// up to @p Use, and return the pair {value, false}. If there is no
+ /// segment before @p Use and there is no "undef" between @p StartIdx and
+ /// @p Use, return {nullptr, false}. If there is an "undef" before @p Use,
+ /// return {nullptr, true}.
+ std::pair<VNInfo*,bool> extendInBlock(ArrayRef<SlotIndex> Undefs,
+ SlotIndex StartIdx, SlotIndex Use);
+
+ /// Simplified version of the above "extendInBlock", which assumes that
+ /// no register lanes are undefined by <def,read-undef> operands.
+ /// If this range is live before @p Use in the basic block that starts
+ /// at @p StartIdx, extend it to be live up to @p Use, and return the
+ /// value. If there is no segment before @p Use, return nullptr.
+ VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Kill);
/// join - Join two live ranges (this, and other) together. This applies
/// mappings to the value numbers in the LHS/RHS ranges as specified. If
@@ -555,6 +578,16 @@ namespace llvm {
return thisIndex < otherIndex;
}
+ /// Returns true if there is an explicit "undef" between @p Begin
+ /// @p End.
+ bool isUndefIn(ArrayRef<SlotIndex> Undefs, SlotIndex Begin,
+ SlotIndex End) const {
+ return std::any_of(Undefs.begin(), Undefs.end(),
+ [Begin,End] (SlotIndex Idx) -> bool {
+ return Begin <= Idx && Idx < End;
+ });
+ }
+
/// Flush segment set into the regular segment vector.
/// The method is to be called after the live range
/// has been created, if use of the segment set was
@@ -581,7 +614,6 @@ namespace llvm {
friend class LiveRangeUpdater;
void addSegmentToSet(Segment S);
void markValNoForDeletion(VNInfo *V);
-
};
inline raw_ostream &operator<<(raw_ostream &OS, const LiveRange &LR) {
@@ -729,6 +761,13 @@ namespace llvm {
weight = llvm::huge_valf;
}
+ /// For a given lane mask @p LaneMask, compute indexes at which the
+ /// lane is marked undefined by subregister <def,read-undef> definitions.
+ void computeSubRangeUndefs(SmallVectorImpl<SlotIndex> &Undefs,
+ LaneBitmask LaneMask,
+ const MachineRegisterInfo &MRI,
+ const SlotIndexes &Indexes) const;
+
bool operator<(const LiveInterval& other) const {
const SlotIndex &thisIndex = beginIndex();
const SlotIndex &otherIndex = other.beginIndex();
Modified: llvm/trunk/include/llvm/CodeGen/LiveIntervalAnalysis.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/LiveIntervalAnalysis.h?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/LiveIntervalAnalysis.h (original)
+++ llvm/trunk/include/llvm/CodeGen/LiveIntervalAnalysis.h Wed Aug 24 08:37:55 2016
@@ -163,17 +163,16 @@ extern cl::opt<bool> UseSegmentSetForPhy
/// LiveInterval::removeEmptySubranges() afterwards.
void shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg);
- /// extendToIndices - Extend the live range of LI to reach all points in
- /// Indices. The points in the Indices array must be jointly dominated by
- /// existing defs in LI. PHI-defs are added as needed to maintain SSA form.
+ /// Extend the live range @p LR to reach all points in @p Indices. The
+ /// points in the @p Indices array must be jointly dominated by existing
+ /// defs in @p LR. PHI-defs are added as needed to maintain SSA form.
///
- /// If a SlotIndex in Indices is the end index of a basic block, LI will be
- /// extended to be live out of the basic block.
+ /// If a SlotIndex in @p Indices is the end index of a basic block, @p LR
+ /// will be extended to be live out of the basic block.
///
/// See also LiveRangeCalc::extend().
void extendToIndices(LiveRange &LR, ArrayRef<SlotIndex> Indices);
-
/// If @p LR has a live value at @p Kill, prune its live range by removing
/// any liveness reachable from Kill. Add live range end points to
/// EndPoints such that extendToIndices(LI, EndPoints) will reconstruct the
Modified: llvm/trunk/lib/CodeGen/LiveInterval.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveInterval.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/LiveInterval.cpp (original)
+++ llvm/trunk/lib/CodeGen/LiveInterval.cpp Wed Aug 24 08:37:55 2016
@@ -59,18 +59,32 @@ public:
typedef LiveRange::Segment Segment;
typedef IteratorT iterator;
- VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator &VNInfoAllocator) {
+ /// A counterpart of LiveRange::createDeadDef: Make sure the range has a
+ /// value defined at @p Def.
+ /// If @p ForVNI is null, and there is no value defined at @p Def, a new
+ /// value will be allocated using @p VNInfoAllocator.
+ /// If @p ForVNI is null, the return value is the value defined at @p Def,
+ /// either a pre-existing one, or the one newly created.
+ /// If @p ForVNI is not null, then @p Def should be the location where
+ /// @p ForVNI is defined. If the range does not have a value defined at
+ /// @p Def, the value @p ForVNI will be used instead of allocating a new
+ /// one. If the range already has a value defined at @p Def, it must be
+ /// same as @p ForVNI. In either case, @p ForVNI will be the return value.
+ VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator *VNInfoAllocator,
+ VNInfo *ForVNI) {
assert(!Def.isDead() && "Cannot define a value at the dead slot");
-
+ assert((!ForVNI || ForVNI->def == Def) &&
+ "If ForVNI is specified, it must match Def");
iterator I = impl().find(Def);
if (I == segments().end()) {
- VNInfo *VNI = LR->getNextValue(Def, VNInfoAllocator);
+ VNInfo *VNI = ForVNI ? ForVNI : LR->getNextValue(Def, *VNInfoAllocator);
impl().insertAtEnd(Segment(Def, Def.getDeadSlot(), VNI));
return VNI;
}
Segment *S = segmentAt(I);
if (SlotIndex::isSameInstr(Def, S->start)) {
+ assert((!ForVNI || ForVNI == S->valno) && "Value number mismatch");
assert(S->valno->def == S->start && "Inconsistent existing value def");
// It is possible to have both normal and early-clobber defs of the same
@@ -84,7 +98,7 @@ public:
return S->valno;
}
assert(SlotIndex::isEarlierInstr(Def, S->start) && "Already live at def");
- VNInfo *VNI = LR->getNextValue(Def, VNInfoAllocator);
+ VNInfo *VNI = ForVNI ? ForVNI : LR->getNextValue(Def, *VNInfoAllocator);
segments().insert(I, Segment(Def, Def.getDeadSlot(), VNI));
return VNI;
}
@@ -93,7 +107,7 @@ public:
if (segments().empty())
return nullptr;
iterator I =
- impl().findInsertPos(Segment(Use.getPrevSlot(), Use, nullptr));
+ impl().findInsertPos(Segment(Use.getPrevSlot(), Use, nullptr));
if (I == segments().begin())
return nullptr;
--I;
@@ -104,6 +118,25 @@ public:
return I->valno;
}
+ std::pair<VNInfo*,bool> extendInBlock(ArrayRef<SlotIndex> Undefs,
+ SlotIndex StartIdx, SlotIndex Use) {
+ if (segments().empty())
+ return std::make_pair(nullptr, false);
+ SlotIndex BeforeUse = Use.getPrevSlot();
+ iterator I = impl().findInsertPos(Segment(BeforeUse, Use, nullptr));
+ if (I == segments().begin())
+ return std::make_pair(nullptr, LR->isUndefIn(Undefs, StartIdx, BeforeUse));
+ --I;
+ if (I->end <= StartIdx)
+ return std::make_pair(nullptr, LR->isUndefIn(Undefs, StartIdx, BeforeUse));
+ if (I->end < Use) {
+ if (LR->isUndefIn(Undefs, I->end, BeforeUse))
+ return std::make_pair(nullptr, true);
+ extendSegmentEndTo(I, Use);
+ }
+ return std::make_pair(I->valno, false);
+ }
+
/// This method is used when we want to extend the segment specified
/// by I to end at the specified endpoint. To do this, we should
/// merge and eliminate all segments that this will overlap
@@ -320,13 +353,20 @@ LiveRange::iterator LiveRange::find(Slot
return I;
}
-VNInfo *LiveRange::createDeadDef(SlotIndex Def,
- VNInfo::Allocator &VNInfoAllocator) {
+VNInfo *LiveRange::createDeadDef(SlotIndex Def, VNInfo::Allocator &VNIAlloc) {
+ // Use the segment set, if it is available.
+ if (segmentSet != nullptr)
+ return CalcLiveRangeUtilSet(this).createDeadDef(Def, &VNIAlloc, nullptr);
+ // Otherwise use the segment vector.
+ return CalcLiveRangeUtilVector(this).createDeadDef(Def, &VNIAlloc, nullptr);
+}
+
+VNInfo *LiveRange::createDeadDef(VNInfo *VNI) {
// Use the segment set, if it is available.
if (segmentSet != nullptr)
- return CalcLiveRangeUtilSet(this).createDeadDef(Def, VNInfoAllocator);
+ return CalcLiveRangeUtilSet(this).createDeadDef(VNI->def, nullptr, VNI);
// Otherwise use the segment vector.
- return CalcLiveRangeUtilVector(this).createDeadDef(Def, VNInfoAllocator);
+ return CalcLiveRangeUtilVector(this).createDeadDef(VNI->def, nullptr, VNI);
}
// overlaps - Return true if the intersection of the two live ranges is
@@ -507,9 +547,15 @@ void LiveRange::append(const Segment S)
segments.push_back(S);
}
-/// extendInBlock - If this range is live before Kill in the basic
-/// block that starts at StartIdx, extend it to be live up to Kill and return
-/// the value. If there is no live range before Kill, return NULL.
+std::pair<VNInfo*,bool> LiveRange::extendInBlock(ArrayRef<SlotIndex> Undefs,
+ SlotIndex StartIdx, SlotIndex Kill) {
+ // Use the segment set, if it is available.
+ if (segmentSet != nullptr)
+ return CalcLiveRangeUtilSet(this).extendInBlock(Undefs, StartIdx, Kill);
+ // Otherwise use the segment vector.
+ return CalcLiveRangeUtilVector(this).extendInBlock(Undefs, StartIdx, Kill);
+}
+
VNInfo *LiveRange::extendInBlock(SlotIndex StartIdx, SlotIndex Kill) {
// Use the segment set, if it is available.
if (segmentSet != nullptr)
@@ -824,6 +870,30 @@ unsigned LiveInterval::getSize() const {
return Sum;
}
+void LiveInterval::computeSubRangeUndefs(SmallVectorImpl<SlotIndex> &Undefs,
+ LaneBitmask LaneMask,
+ const MachineRegisterInfo &MRI,
+ const SlotIndexes &Indexes) const {
+ assert(TargetRegisterInfo::isVirtualRegister(reg));
+ LaneBitmask VRegMask = MRI.getMaxLaneMaskForVReg(reg);
+ assert((VRegMask & LaneMask) != 0);
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+ for (const MachineOperand &MO : MRI.def_operands(reg)) {
+ if (!MO.isUndef())
+ continue;
+ unsigned SubReg = MO.getSubReg();
+ assert(SubReg != 0 && "Undef should only be set on subreg defs");
+ LaneBitmask DefMask = TRI.getSubRegIndexLaneMask(SubReg);
+ LaneBitmask UndefMask = VRegMask & ~DefMask;
+ if ((UndefMask & LaneMask) != 0) {
+ const MachineInstr &MI = *MO.getParent();
+ bool EarlyClobber = MO.isEarlyClobber();
+ SlotIndex Pos = Indexes.getInstructionIndex(MI).getRegSlot(EarlyClobber);
+ Undefs.push_back(Pos);
+ }
+ }
+}
+
raw_ostream& llvm::operator<<(raw_ostream& os, const LiveRange::Segment &S) {
return os << '[' << S.start << ',' << S.end << ':' << S.valno->id << ')';
}
Modified: llvm/trunk/lib/CodeGen/LiveIntervalAnalysis.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveIntervalAnalysis.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/LiveIntervalAnalysis.cpp (original)
+++ llvm/trunk/lib/CodeGen/LiveIntervalAnalysis.cpp Wed Aug 24 08:37:55 2016
@@ -504,8 +504,7 @@ bool LiveIntervals::computeDeadValues(Li
return MayHaveSplitComponents;
}
-void LiveIntervals::shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg)
-{
+void LiveIntervals::shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg) {
DEBUG(dbgs() << "Shrink: " << SR << '\n');
assert(TargetRegisterInfo::isVirtualRegister(Reg)
&& "Can only shrink virtual registers");
@@ -516,12 +515,14 @@ void LiveIntervals::shrinkToUses(LiveInt
SlotIndex LastIdx;
for (MachineOperand &MO : MRI->reg_operands(Reg)) {
MachineInstr *UseMI = MO.getParent();
- if (UseMI->isDebugValue())
+ if (UseMI->isDebugValue() || !MO.readsReg())
continue;
// Maybe the operand is for a subregister we don't care about.
unsigned SubReg = MO.getSubReg();
if (SubReg != 0) {
LaneBitmask LaneMask = TRI->getSubRegIndexLaneMask(SubReg);
+ if (MO.isDef())
+ LaneMask = ~LaneMask & MRI->getMaxLaneMaskForVReg(Reg);
if ((LaneMask & SR.LaneMask) == 0)
continue;
}
@@ -578,7 +579,7 @@ void LiveIntervals::extendToIndices(Live
assert(LRCalc && "LRCalc not initialized.");
LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
for (unsigned i = 0, e = Indices.size(); i != e; ++i)
- LRCalc->extend(LR, Indices[i]);
+ LRCalc->extend(LR, Indices[i], /*PhysReg=*/0, /*Undefs=*/{});
}
void LiveIntervals::pruneValue(LiveRange &LR, SlotIndex Kill,
@@ -954,7 +955,8 @@ public:
LiveInterval &LI = LIS.getInterval(Reg);
if (LI.hasSubRanges()) {
unsigned SubReg = MO.getSubReg();
- LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubReg);
+ LaneBitmask LaneMask = SubReg ? TRI.getSubRegIndexLaneMask(SubReg)
+ : MRI.getMaxLaneMaskForVReg(Reg);
for (LiveInterval::SubRange &S : LI.subranges()) {
if ((S.LaneMask & LaneMask) == 0)
continue;
@@ -1545,15 +1547,19 @@ void LiveIntervals::removePhysRegDefAt(u
}
void LiveIntervals::removeVRegDefAt(LiveInterval &LI, SlotIndex Pos) {
+ // LI may not have the main range computed yet, but its subranges may
+ // be present.
VNInfo *VNI = LI.getVNInfoAt(Pos);
- if (VNI == nullptr)
- return;
- LI.removeValNo(VNI);
+ if (VNI != nullptr) {
+ assert(VNI->def.getBaseIndex() == Pos.getBaseIndex());
+ LI.removeValNo(VNI);
+ }
- // Also remove the value in subranges.
+ // Also remove the value defined in subranges.
for (LiveInterval::SubRange &S : LI.subranges()) {
if (VNInfo *SVNI = S.getVNInfoAt(Pos))
- S.removeValNo(SVNI);
+ if (SVNI->def.getBaseIndex() == Pos.getBaseIndex())
+ S.removeValNo(SVNI);
}
LI.removeEmptySubRanges();
}
Modified: llvm/trunk/lib/CodeGen/LiveRangeCalc.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveRangeCalc.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/LiveRangeCalc.cpp (original)
+++ llvm/trunk/lib/CodeGen/LiveRangeCalc.cpp Wed Aug 24 08:37:55 2016
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "LiveRangeCalc.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -23,6 +24,7 @@ void LiveRangeCalc::resetLiveOutMap() {
unsigned NumBlocks = MF->getNumBlockIDs();
Seen.clear();
Seen.resize(NumBlocks);
+ EntryInfoMap.clear();
Map.resize(NumBlocks);
}
@@ -64,9 +66,9 @@ void LiveRangeCalc::calculate(LiveInterv
unsigned SubReg = MO.getSubReg();
if (LI.hasSubRanges() || (SubReg != 0 && TrackSubRegs)) {
- LaneBitmask Mask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg)
- : MRI->getMaxLaneMaskForVReg(Reg);
-
+ LaneBitmask WholeMask = MRI->getMaxLaneMaskForVReg(Reg);
+ LaneBitmask SubMask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg)
+ : WholeMask;
// If this is the first time we see a subregister def, initialize
// subranges by creating a copy of the main range.
if (!LI.hasSubRanges() && !LI.empty()) {
@@ -74,17 +76,18 @@ void LiveRangeCalc::calculate(LiveInterv
LI.createSubRangeFrom(*Alloc, ClassMask, LI);
}
+ LaneBitmask Mask = SubMask;
for (LiveInterval::SubRange &S : LI.subranges()) {
// A Mask for subregs common to the existing subrange and current def.
LaneBitmask Common = S.LaneMask & Mask;
if (Common == 0)
continue;
- // A Mask for subregs covered by the subrange but not the current def.
- LaneBitmask LRest = S.LaneMask & ~Mask;
LiveInterval::SubRange *CommonRange;
- if (LRest != 0) {
- // Split current subrange into Common and LRest ranges.
- S.LaneMask = LRest;
+ // A Mask for subregs covered by the subrange but not the current def.
+ if (LaneBitmask RM = S.LaneMask & ~Mask) {
+ // Split the subrange S into two parts: one covered by the current
+ // def (CommonRange), and the one not affected by it (updated S).
+ S.LaneMask = RM;
CommonRange = LI.createSubRangeFrom(*Alloc, Common, S);
} else {
assert(Common == S.LaneMask);
@@ -116,8 +119,9 @@ void LiveRangeCalc::calculate(LiveInterv
// necessary.
if (LI.hasSubRanges()) {
for (LiveInterval::SubRange &S : LI.subranges()) {
- resetLiveOutMap();
- extendToUses(S, Reg, S.LaneMask);
+ LiveRangeCalc SubLRC;
+ SubLRC.reset(MF, Indexes, DomTree, Alloc);
+ SubLRC.extendToUses(S, Reg, S.LaneMask, &LI);
}
LI.clear();
constructMainRangeFromSubranges(LI);
@@ -139,9 +143,8 @@ void LiveRangeCalc::constructMainRangeFr
MainRange.createDeadDef(VNI->def, *Alloc);
}
}
-
resetLiveOutMap();
- extendToUses(MainRange, LI.reg);
+ extendToUses(MainRange, LI.reg, ~0U, &LI);
}
void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) {
@@ -154,8 +157,12 @@ void LiveRangeCalc::createDeadDefs(LiveR
}
-void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg,
- LaneBitmask Mask) {
+void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask Mask,
+ LiveInterval *LI) {
+ SmallVector<SlotIndex, 4> Undefs;
+ if (LI != nullptr)
+ LI->computeSubRangeUndefs(Undefs, Mask, *MRI, *Indexes);
+
// Visit all operands that read Reg. This may include partial defs.
const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) {
@@ -163,20 +170,16 @@ void LiveRangeCalc::extendToUses(LiveRan
// by LiveIntervalAnalysis::addKillFlags().
if (MO.isUse())
MO.setIsKill(false);
- else {
- // We only care about uses, but on the main range (mask ~0u) this includes
- // the "virtual" reads happening for subregister defs.
- if (Mask != ~0u)
- continue;
- }
-
if (!MO.readsReg())
continue;
+
unsigned SubReg = MO.getSubReg();
if (SubReg != 0) {
- LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(SubReg);
+ LaneBitmask SLM = TRI.getSubRegIndexLaneMask(SubReg);
+ if (MO.isDef())
+ SLM = MRI->getMaxLaneMaskForVReg(Reg) & ~SLM;
// Ignore uses not covering the current subrange.
- if ((SubRegMask & Mask) == 0)
+ if ((SLM & Mask) == 0)
continue;
}
@@ -205,7 +208,7 @@ void LiveRangeCalc::extendToUses(LiveRan
// MI is reading Reg. We may have visited MI before if it happens to be
// reading Reg multiple times. That is OK, extend() is idempotent.
- extend(LR, UseIdx, Reg);
+ extend(LR, UseIdx, Reg, Undefs);
}
}
@@ -235,8 +238,8 @@ void LiveRangeCalc::updateFromLiveIns()
LiveIn.clear();
}
-
-void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg) {
+void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs) {
assert(Use.isValid() && "Invalid SlotIndex");
assert(Indexes && "Missing SlotIndexes");
assert(DomTree && "Missing dominator tree");
@@ -245,14 +248,15 @@ void LiveRangeCalc::extend(LiveRange &LR
assert(UseMBB && "No MBB at Use");
// Is there a def in the same MBB we can extend?
- if (LR.extendInBlock(Indexes->getMBBStartIdx(UseMBB), Use))
+ auto EP = LR.extendInBlock(Undefs, Indexes->getMBBStartIdx(UseMBB), Use);
+ if (EP.first != nullptr || EP.second)
return;
// Find the single reaching def, or determine if Use is jointly dominated by
// multiple values, and we may need to create even more phi-defs to preserve
// VNInfo SSA form. Perform a search for all predecessor blocks where we
// know the dominating VNInfo.
- if (findReachingDefs(LR, *UseMBB, Use, PhysReg))
+ if (findReachingDefs(LR, *UseMBB, Use, PhysReg, Undefs))
return;
// When there were multiple different values, we may need new PHIs.
@@ -271,8 +275,72 @@ void LiveRangeCalc::calculateValues() {
}
+bool LiveRangeCalc::isDefOnEntry(LiveRange &LR, ArrayRef<SlotIndex> Undefs,
+ MachineBasicBlock &MBB, BitVector &DefOnEntry,
+ BitVector &UndefOnEntry) {
+ unsigned BN = MBB.getNumber();
+ if (DefOnEntry[BN])
+ return true;
+ if (UndefOnEntry[BN])
+ return false;
+
+ auto MarkDefined =
+ [this,BN,&DefOnEntry,&UndefOnEntry] (MachineBasicBlock &B) -> bool {
+ for (MachineBasicBlock *S : B.successors())
+ DefOnEntry[S->getNumber()] = true;
+ DefOnEntry[BN] = true;
+ return true;
+ };
+
+ SetVector<unsigned> WorkList;
+ // Checking if the entry of MBB is reached by some def: add all predecessors
+ // that are potentially defined-on-exit to the work list.
+ for (MachineBasicBlock *P : MBB.predecessors())
+ WorkList.insert(P->getNumber());
+
+ for (unsigned i = 0; i != WorkList.size(); ++i) {
+ // Determine if the exit from the block is reached by some def.
+ unsigned N = WorkList[i];
+ MachineBasicBlock &B = *MF->getBlockNumbered(N);
+ if (Seen[N] && Map[&B].first != nullptr)
+ return MarkDefined(B);
+ SlotIndex Begin, End;
+ std::tie(Begin, End) = Indexes->getMBBRange(&B);
+ LiveRange::iterator UB = std::upper_bound(LR.begin(), LR.end(), End);
+ if (UB != LR.begin()) {
+ LiveRange::Segment &Seg = *std::prev(UB);
+ if (Seg.end > Begin) {
+ // There is a segment that overlaps B. If the range is not explicitly
+ // undefined between the end of the segment and the end of the block,
+ // treat the block as defined on exit. If it is, go to the next block
+ // on the work list.
+ if (LR.isUndefIn(Undefs, Seg.end, End))
+ continue;
+ return MarkDefined(B);
+ }
+ }
+
+ // No segment overlaps with this block. If this block is not defined on
+ // entry, or it undefines the range, do not process its predecessors.
+ if (UndefOnEntry[N] || LR.isUndefIn(Undefs, Begin, End)) {
+ UndefOnEntry[N] = true;
+ continue;
+ }
+ if (DefOnEntry[N])
+ return MarkDefined(B);
+
+ // Still don't know: add all predecessors to the work list.
+ for (MachineBasicBlock *P : B.predecessors())
+ WorkList.insert(P->getNumber());
+ }
+
+ UndefOnEntry[BN] = true;
+ return false;
+}
+
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
- SlotIndex Use, unsigned PhysReg) {
+ SlotIndex Use, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs) {
unsigned UseMBBNum = UseMBB.getNumber();
// Block numbers where LR should be live-in.
@@ -282,12 +350,14 @@ bool LiveRangeCalc::findReachingDefs(Liv
bool UniqueVNI = true;
VNInfo *TheVNI = nullptr;
+ bool FoundUndef = false;
+
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]);
#ifndef NDEBUG
- if (MBB->pred_empty()) {
+ if (Undefs.size() > 0 && MBB->pred_empty()) {
MBB->getParent()->verify();
errs() << "Use of " << PrintReg(PhysReg)
<< " does not have a corresponding definition on every path:\n";
@@ -306,6 +376,7 @@ bool LiveRangeCalc::findReachingDefs(Liv
llvm_unreachable("Invalid global physical register");
}
#endif
+ FoundUndef |= MBB->pred_empty();
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
@@ -326,18 +397,21 @@ bool LiveRangeCalc::findReachingDefs(Liv
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
- VNInfo *VNI = LR.extendInBlock(Start, End);
+ auto EP = LR.extendInBlock(Undefs, Start, End);
+ VNInfo *VNI = EP.first;
+ FoundUndef |= EP.second;
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
- continue;
}
+ if (VNI || EP.second)
+ continue;
// No, we need a live-in value for Pred as well
if (Pred != &UseMBB)
- WorkList.push_back(Pred->getNumber());
+ WorkList.push_back(Pred->getNumber());
else
// Loopback to UseMBB, so value is really live through.
Use = SlotIndex();
@@ -345,6 +419,9 @@ bool LiveRangeCalc::findReachingDefs(Liv
}
LiveIn.clear();
+ FoundUndef |= (TheVNI == nullptr);
+ if (Undefs.size() > 0 && FoundUndef)
+ UniqueVNI = false;
// Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but
// neither require it. Skip the sorting overhead for small updates.
@@ -353,27 +430,39 @@ bool LiveRangeCalc::findReachingDefs(Liv
// If a unique reaching def was found, blit in the live ranges immediately.
if (UniqueVNI) {
+ assert(TheVNI != nullptr);
LiveRangeUpdater Updater(&LR);
- for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(),
- E = WorkList.end(); I != E; ++I) {
- SlotIndex Start, End;
- std::tie(Start, End) = Indexes->getMBBRange(*I);
- // Trim the live range in UseMBB.
- if (*I == UseMBBNum && Use.isValid())
- End = Use;
- else
- Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr);
- Updater.add(Start, End, TheVNI);
+ for (unsigned BN : WorkList) {
+ SlotIndex Start, End;
+ std::tie(Start, End) = Indexes->getMBBRange(BN);
+ // Trim the live range in UseMBB.
+ if (BN == UseMBBNum && Use.isValid())
+ End = Use;
+ else
+ Map[MF->getBlockNumbered(BN)] = LiveOutPair(TheVNI, nullptr);
+ Updater.add(Start, End, TheVNI);
}
return true;
}
+ // Prepare the defined/undefined bit vectors.
+ auto EF = EntryInfoMap.find(&LR);
+ if (EF == EntryInfoMap.end()) {
+ unsigned N = MF->getNumBlockIDs();
+ EF = EntryInfoMap.insert({&LR, {BitVector(), BitVector()}}).first;
+ EF->second.first.resize(N);
+ EF->second.second.resize(N);
+ }
+ BitVector &DefOnEntry = EF->second.first;
+ BitVector &UndefOnEntry = EF->second.second;
+
// Multiple values were found, so transfer the work list to the LiveIn array
// where UpdateSSA will use it as a work list.
LiveIn.reserve(WorkList.size());
- for (SmallVectorImpl<unsigned>::const_iterator
- I = WorkList.begin(), E = WorkList.end(); I != E; ++I) {
- MachineBasicBlock *MBB = MF->getBlockNumbered(*I);
+ for (unsigned BN : WorkList) {
+ MachineBasicBlock *MBB = MF->getBlockNumbered(BN);
+ if (Undefs.size() > 0 && !isDefOnEntry(LR, Undefs, *MBB, DefOnEntry, UndefOnEntry))
+ continue;
addLiveInBlock(LR, DomTree->getNode(MBB));
if (MBB == &UseMBB)
LiveIn.back().Kill = Use;
@@ -458,10 +547,12 @@ void LiveRangeCalc::updateSSA() {
I.DomNode = nullptr;
// Add liveness since updateFromLiveIns now skips this node.
- if (I.Kill.isValid())
- LR.addSegment(LiveInterval::Segment(Start, I.Kill, VNI));
- else {
- LR.addSegment(LiveInterval::Segment(Start, End, VNI));
+ if (I.Kill.isValid()) {
+ if (VNI)
+ LR.addSegment(LiveInterval::Segment(Start, I.Kill, VNI));
+ } else {
+ if (VNI)
+ LR.addSegment(LiveInterval::Segment(Start, End, VNI));
LOP = LiveOutPair(VNI, Node);
}
} else if (IDomValue.first) {
Modified: llvm/trunk/lib/CodeGen/LiveRangeCalc.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveRangeCalc.h?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/LiveRangeCalc.h (original)
+++ llvm/trunk/lib/CodeGen/LiveRangeCalc.h Wed Aug 24 08:37:55 2016
@@ -22,6 +22,7 @@
#ifndef LLVM_LIB_CODEGEN_LIVERANGECALC_H
#define LLVM_LIB_CODEGEN_LIVERANGECALC_H
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/IndexedMap.h"
#include "llvm/CodeGen/LiveInterval.h"
@@ -53,6 +54,19 @@ class LiveRangeCalc {
/// when switching live ranges.
BitVector Seen;
+ /// Map LiveRange to sets of blocks (represented by bit vectors) that
+ /// in the live range are defined on entry and undefined on entry.
+ /// A block is defined on entry if there is a path from at least one of
+ /// the defs in the live range to the entry of the block, and conversely,
+ /// a block is undefined on entry, if there is no such path (i.e. no
+ /// definition reaches the entry of the block). A single LiveRangeCalc
+ /// object is used to track live-out information for multiple registers
+ /// in live range splitting (which is ok, since the live ranges of these
+ /// registers do not overlap), but the defined/undefined information must
+ /// be kept separate for each individual range.
+ /// By convention, EntryInfoMap[&LR] = { Defined, Undefined }.
+ std::map<LiveRange*,std::pair<BitVector,BitVector>> EntryInfoMap;
+
/// Map each basic block where a live range is live out to the live-out value
/// and its defining block.
///
@@ -101,18 +115,31 @@ class LiveRangeCalc {
/// used to add entries directly.
SmallVector<LiveInBlock, 16> LiveIn;
- /// Assuming that @p LR is live-in to @p UseMBB, find the set of defs that can
- /// reach it.
- ///
- /// If only one def can reach @p UseMBB, all paths from the def to @p UseMBB
- /// are added to @p LR, and the function returns true.
+ /// Check if the entry to block @p MBB can be reached by any of the defs
+ /// in @p LR. Return true if none of the defs reach the entry to @p MBB.
+ bool isDefOnEntry(LiveRange &LR, ArrayRef<SlotIndex> Undefs,
+ MachineBasicBlock &MBB, BitVector &DefOnEntry,
+ BitVector &UndefOnEntry);
+
+ /// Find the set of defs that can reach @p Kill. @p Kill must belong to
+ /// @p UseMBB.
+ ///
+ /// If exactly one def can reach @p UseMBB, and the def dominates @p Kill,
+ /// all paths from the def to @p UseMBB are added to @p LR, and the function
+ /// returns true.
///
/// If multiple values can reach @p UseMBB, the blocks that need @p LR to be
/// live in are added to the LiveIn array, and the function returns false.
///
+ /// The array @p Undef provides the locations where the range @p LR becomes
+ /// undefined by <def,read-undef> operands on other subranges. If @p Undef
+ /// is non-empty and @p Kill is jointly dominated only by the entries of
+ /// @p Undef, the function returns false.
+ ///
/// PhysReg, when set, is used to verify live-in lists on basic blocks.
bool findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
- SlotIndex Kill, unsigned PhysReg);
+ SlotIndex Kill, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs);
/// updateSSA - Compute the values that will be live in to all requested
/// blocks in LiveIn. Create PHI-def values as required to preserve SSA form.
@@ -127,9 +154,14 @@ class LiveRangeCalc {
/// Extend the live range of @p LR to reach all uses of Reg.
///
- /// All uses must be jointly dominated by existing liveness. PHI-defs are
- /// inserted as needed to preserve SSA form.
- void extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask LaneMask);
+ /// If @p LR is a main range, or if @p LI is null, then all uses must be
+ /// jointly dominated by the definitions from @p LR. If @p LR is a subrange
+ /// of the live interval @p LI, corresponding to lane mask @p LaneMask,
+ /// all uses must be jointly dominated by the definitions from @p LR
+ /// together with definitions of other lanes where @p LR becomes undefined
+ /// (via <def,read-undef> operands).
+ void extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask LaneMask,
+ LiveInterval *LI = nullptr);
/// Reset Map and Seen fields.
void resetLiveOutMap();
@@ -169,7 +201,8 @@ public:
/// inserted as required to preserve SSA form.
///
/// PhysReg, when set, is used to verify live-in lists on basic blocks.
- void extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg = 0);
+ void extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs);
/// createDeadDefs - Create a dead def in LI for every def operand of Reg.
/// Each instruction defining Reg gets a new VNInfo with a corresponding
Modified: llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp (original)
+++ llvm/trunk/lib/CodeGen/LiveRangeEdit.cpp Wed Aug 24 08:37:55 2016
@@ -37,6 +37,13 @@ LiveInterval &LiveRangeEdit::createEmpty
VRM->setIsSplitFromReg(VReg, VRM->getOriginal(OldReg));
}
LiveInterval &LI = LIS.createEmptyInterval(VReg);
+ // Create empty subranges if the OldReg's interval has them. Do not create
+ // the main range here---it will be constructed later after the subranges
+ // have been finalized.
+ LiveInterval &OldLI = LIS.getInterval(OldReg);
+ VNInfo::Allocator &Alloc = LIS.getVNInfoAllocator();
+ for (LiveInterval::SubRange &S : OldLI.subranges())
+ LI.createSubRange(Alloc, S.LaneMask);
return LI;
}
@@ -66,6 +73,8 @@ void LiveRangeEdit::scanRemattable(Alias
unsigned Original = VRM->getOriginal(getReg());
LiveInterval &OrigLI = LIS.getInterval(Original);
VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def);
+ if (!OrigVNI)
+ continue;
MachineInstr *DefMI = LIS.getInstructionFromIndex(OrigVNI->def);
if (!DefMI)
continue;
@@ -335,6 +344,7 @@ void LiveRangeEdit::eliminateDeadDef(Mac
// allocations of the func are done.
if (isOrigDef && DeadRemats && TII.isTriviallyReMaterializable(*MI, AA)) {
LiveInterval &NewLI = createEmptyIntervalFrom(Dest);
+ NewLI.removeEmptySubRanges();
VNInfo *VNI = NewLI.getNextValue(Idx, LIS.getVNInfoAllocator());
NewLI.addSegment(LiveInterval::Segment(Idx, Idx.getDeadSlot(), VNI));
pop_back();
Modified: llvm/trunk/lib/CodeGen/MachineVerifier.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/MachineVerifier.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/MachineVerifier.cpp (original)
+++ llvm/trunk/lib/CodeGen/MachineVerifier.cpp Wed Aug 24 08:37:55 2016
@@ -1813,18 +1813,25 @@ void MachineVerifier::verifyLiveRangeSeg
bool hasRead = false;
bool hasSubRegDef = false;
bool hasDeadDef = false;
+ LaneBitmask RLM = MRI->getMaxLaneMaskForVReg(Reg);
for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
if (!MOI->isReg() || MOI->getReg() != Reg)
continue;
- if (LaneMask != 0 &&
- (LaneMask & TRI->getSubRegIndexLaneMask(MOI->getSubReg())) == 0)
- continue;
+ unsigned Sub = MOI->getSubReg();
+ LaneBitmask SLM = Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : RLM;
if (MOI->isDef()) {
- if (MOI->getSubReg() != 0)
+ if (Sub != 0) {
hasSubRegDef = true;
+ // An operand vreg0:sub0<def> reads vreg0:sub1..n. Invert the lane
+ // mask for subregister defs. Read-undef defs will be handled by
+ // readsReg below.
+ SLM = ~SLM & RLM;
+ }
if (MOI->isDead())
hasDeadDef = true;
}
+ if (LaneMask != 0 && !(LaneMask & SLM))
+ continue;
if (MOI->readsReg())
hasRead = true;
}
Modified: llvm/trunk/lib/CodeGen/RegAllocBase.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegAllocBase.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/RegAllocBase.cpp (original)
+++ llvm/trunk/lib/CodeGen/RegAllocBase.cpp Wed Aug 24 08:37:55 2016
@@ -143,6 +143,7 @@ void RegAllocBase::allocatePhysRegs() {
continue;
}
DEBUG(dbgs() << "queuing new interval: " << *SplitVirtReg << "\n");
+ assert(!SplitVirtReg->empty() && "expecting non-empty interval");
assert(TargetRegisterInfo::isVirtualRegister(SplitVirtReg->reg) &&
"expect split value in virtual register");
enqueue(SplitVirtReg);
Modified: llvm/trunk/lib/CodeGen/RegisterCoalescer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RegisterCoalescer.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/RegisterCoalescer.cpp (original)
+++ llvm/trunk/lib/CodeGen/RegisterCoalescer.cpp Wed Aug 24 08:37:55 2016
@@ -1211,6 +1211,17 @@ bool RegisterCoalescer::eliminateUndefCo
MO.setIsUndef(true);
DEBUG(dbgs() << "\tnew undef: " << UseIdx << '\t' << MI);
}
+
+ // A def of a subregister may be a use of the other subregisters, so
+ // deleting a def of a subregister may also remove uses. Since CopyMI
+ // is still part of the function (but about to be erased), mark all
+ // defs of DstReg in it as <undef>, so that shrinkToUses would
+ // ignore them.
+ for (MachineOperand &MO : CopyMI->operands())
+ if (MO.isReg() && MO.isDef() && MO.getReg() == DstReg)
+ MO.setIsUndef(true);
+ LIS->shrinkToUses(&DstLI);
+
return true;
}
@@ -1890,12 +1901,22 @@ public:
/// no useful information and can be removed.
void pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask);
+ /// Pruning values in subranges can lead to removing segments in these
+ /// subranges started by IMPLICIT_DEFs. The corresponding segments in
+ /// the main range also need to be removed. This function will mark
+ /// the corresponding values in the main range as pruned, so that
+ /// eraseInstrs can do the final cleanup.
+ /// The parameter @p LI must be the interval whose main range is the
+ /// live range LR.
+ void pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange);
+
/// Erase any machine instructions that have been coalesced away.
/// Add erased instructions to ErasedInstrs.
/// Add foreign virtual registers to ShrinkRegs if their live range ended at
/// the erased instrs.
void eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
- SmallVectorImpl<unsigned> &ShrinkRegs);
+ SmallVectorImpl<unsigned> &ShrinkRegs,
+ LiveInterval *LI = nullptr);
/// Remove liverange defs at places where implicit defs will be removed.
void removeImplicitDefs();
@@ -2416,7 +2437,8 @@ void JoinVals::pruneValues(JoinVals &Oth
for (MachineOperand &MO :
Indexes->getInstructionFromIndex(Def)->operands()) {
if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) {
- MO.setIsUndef(EraseImpDef);
+ if (MO.getSubReg() != 0)
+ MO.setIsUndef(EraseImpDef);
MO.setIsDead(false);
}
}
@@ -2449,8 +2471,7 @@ void JoinVals::pruneValues(JoinVals &Oth
}
}
-void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask)
-{
+void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) {
// Look for values being erased.
bool DidPrune = false;
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
@@ -2487,6 +2508,30 @@ void JoinVals::pruneSubRegValues(LiveInt
LI.removeEmptySubRanges();
}
+/// Check if any of the subranges of @p LI contain a definition at @p Def.
+static bool isDefInSubRange(LiveInterval &LI, SlotIndex Def) {
+ for (LiveInterval::SubRange &SR : LI.subranges()) {
+ if (VNInfo *VNI = SR.Query(Def).valueOutOrDead())
+ if (VNI->def == Def)
+ return true;
+ }
+ return false;
+}
+
+void JoinVals::pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange) {
+ assert(&static_cast<LiveRange&>(LI) == &LR);
+
+ for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
+ if (Vals[i].Resolution != CR_Keep)
+ continue;
+ VNInfo *VNI = LR.getValNumInfo(i);
+ if (VNI->isUnused() || VNI->isPHIDef() || isDefInSubRange(LI, VNI->def))
+ continue;
+ Vals[i].Pruned = true;
+ ShrinkMainRange = true;
+ }
+}
+
void JoinVals::removeImplicitDefs() {
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
Val &V = Vals[i];
@@ -2500,7 +2545,8 @@ void JoinVals::removeImplicitDefs() {
}
void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
- SmallVectorImpl<unsigned> &ShrinkRegs) {
+ SmallVectorImpl<unsigned> &ShrinkRegs,
+ LiveInterval *LI) {
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
// Get the def location before markUnused() below invalidates it.
SlotIndex Def = LR.getValNumInfo(i)->def;
@@ -2512,12 +2558,64 @@ void JoinVals::eraseInstrs(SmallPtrSetIm
if (!Vals[i].ErasableImplicitDef || !Vals[i].Pruned)
break;
// Remove value number i from LR.
+ // For intervals with subranges, removing a segment from the main range
+ // may require extending the previous segment: for each definition of
+ // a subregister, there will be a corresponding def in the main range.
+ // That def may fall in the middle of a segment from another subrange.
+ // In such cases, removing this def from the main range must be
+ // complemented by extending the main range to account for the liveness
+ // of the other subrange.
VNInfo *VNI = LR.getValNumInfo(i);
+ SlotIndex Def = VNI->def;
+ // The new end point of the main range segment to be extended.
+ SlotIndex NewEnd;
+ if (LI != nullptr) {
+ LiveRange::iterator I = LR.FindSegmentContaining(Def);
+ assert(I != LR.end());
+ // Do not extend beyond the end of the segment being removed.
+ // The segment may have been pruned in preparation for joining
+ // live ranges.
+ NewEnd = I->end;
+ }
+
LR.removeValNo(VNI);
// Note that this VNInfo is reused and still referenced in NewVNInfo,
// make it appear like an unused value number.
VNI->markUnused();
- DEBUG(dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n');
+
+ if (LI != nullptr && LI->hasSubRanges()) {
+ assert(static_cast<LiveRange*>(LI) == &LR);
+ // Determine the end point based on the subrange information:
+ // minimum of (earliest def of next segment,
+ // latest end point of containing segment)
+ SlotIndex ED, LE;
+ for (LiveInterval::SubRange &SR : LI->subranges()) {
+ LiveRange::iterator I = SR.find(Def);
+ if (I == SR.end())
+ continue;
+ if (I->start > Def)
+ ED = ED.isValid() ? std::min(ED, I->start) : I->start;
+ else
+ LE = LE.isValid() ? std::max(LE, I->end) : I->end;
+ }
+ if (LE.isValid())
+ NewEnd = std::min(NewEnd, LE);
+ if (ED.isValid())
+ NewEnd = std::min(NewEnd, ED);
+
+ // We only want to do the extension if there was a subrange that
+ // was live across Def.
+ if (LE.isValid()) {
+ LiveRange::iterator S = LR.find(Def);
+ if (S != LR.begin())
+ std::prev(S)->end = NewEnd;
+ }
+ }
+ DEBUG({
+ dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n';
+ if (LI != nullptr)
+ dbgs() << "\t\t LHS = " << *LI << '\n';
+ });
LLVM_FALLTHROUGH;
}
@@ -2698,6 +2796,10 @@ bool RegisterCoalescer::joinVirtRegs(Coa
}
DEBUG(dbgs() << "\tJoined SubRanges " << LHS << "\n");
+ // Pruning implicit defs from subranges may result in the main range
+ // having stale segments.
+ LHSVals.pruneMainSegments(LHS, ShrinkMainRange);
+
LHSVals.pruneSubRegValues(LHS, ShrinkMask);
RHSVals.pruneSubRegValues(LHS, ShrinkMask);
}
@@ -2713,7 +2815,7 @@ bool RegisterCoalescer::joinVirtRegs(Coa
// Erase COPY and IMPLICIT_DEF instructions. This may cause some external
// registers to require trimming.
SmallVector<unsigned, 8> ShrinkRegs;
- LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
+ LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs, &LHS);
RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
while (!ShrinkRegs.empty())
shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val()));
Modified: llvm/trunk/lib/CodeGen/RenameIndependentSubregs.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/RenameIndependentSubregs.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/RenameIndependentSubregs.cpp (original)
+++ llvm/trunk/lib/CodeGen/RenameIndependentSubregs.cpp Wed Aug 24 08:37:55 2016
@@ -353,6 +353,11 @@ void RenameIndependentSubregs::computeMa
if (I == 0)
LI.clear();
LIS->constructMainRangeFromSubranges(LI);
+ // A def of a subregister may be a use of other register lanes. Replacing
+ // such a def with a def of a different register will eliminate the use,
+ // and may cause the recorded live range to be larger than the actual
+ // liveness in the program IR.
+ LIS->shrinkToUses(&LI);
}
}
Modified: llvm/trunk/lib/CodeGen/SplitKit.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SplitKit.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SplitKit.cpp (original)
+++ llvm/trunk/lib/CodeGen/SplitKit.cpp Wed Aug 24 08:37:55 2016
@@ -381,9 +381,59 @@ LLVM_DUMP_METHOD void SplitEditor::dump(
}
#endif
+LiveInterval::SubRange &SplitEditor::getSubRangeForMask(LaneBitmask LM,
+ LiveInterval &LI) {
+ for (LiveInterval::SubRange &S : LI.subranges())
+ if (S.LaneMask == LM)
+ return S;
+ llvm_unreachable("SubRange for this mask not found");
+};
+
+void SplitEditor::addDeadDef(LiveInterval &LI, VNInfo *VNI, bool Original) {
+ if (!LI.hasSubRanges()) {
+ LI.createDeadDef(VNI);
+ return;
+ }
+
+ SlotIndex Def = VNI->def;
+ if (Original) {
+ // If we are transferring a def from the original interval, make sure
+ // to only update the subranges for which the original subranges had
+ // a def at this location.
+ for (LiveInterval::SubRange &S : LI.subranges()) {
+ auto &PS = getSubRangeForMask(S.LaneMask, Edit->getParent());
+ VNInfo *PV = PS.getVNInfoAt(Def);
+ if (PV != nullptr && PV->def == Def)
+ S.createDeadDef(Def, LIS.getVNInfoAllocator());
+ }
+ } else {
+ // This is a new def: either from rematerialization, or from an inserted
+ // copy. Since rematerialization can regenerate a definition of a sub-
+ // register, we need to check which subranges need to be updated.
+ const MachineInstr *DefMI = LIS.getInstructionFromIndex(Def);
+ assert(DefMI != nullptr);
+ LaneBitmask LM = 0;
+ for (const MachineOperand &DefOp : DefMI->defs()) {
+ unsigned R = DefOp.getReg();
+ if (R != LI.reg)
+ continue;
+ if (unsigned SR = DefOp.getSubReg())
+ LM |= TRI.getSubRegIndexLaneMask(SR);
+ else {
+ LM = MRI.getMaxLaneMaskForVReg(R);
+ break;
+ }
+ }
+ for (LiveInterval::SubRange &S : LI.subranges())
+ if (S.LaneMask & LM)
+ S.createDeadDef(Def, LIS.getVNInfoAllocator());
+ }
+}
+
VNInfo *SplitEditor::defValue(unsigned RegIdx,
const VNInfo *ParentVNI,
- SlotIndex Idx) {
+ SlotIndex Idx,
+ bool Original) {
assert(ParentVNI && "Mapping NULL value");
assert(Idx.isValid() && "Invalid SlotIndex");
assert(Edit->getParent().getVNInfoAt(Idx) == ParentVNI && "Bad Parent VNI");
@@ -392,28 +442,28 @@ VNInfo *SplitEditor::defValue(unsigned R
// Create a new value.
VNInfo *VNI = LI->getNextValue(Idx, LIS.getVNInfoAllocator());
+ bool Force = LI->hasSubRanges();
+ ValueForcePair FP(Force ? nullptr : VNI, Force);
// Use insert for lookup, so we can add missing values with a second lookup.
std::pair<ValueMap::iterator, bool> InsP =
- Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id),
- ValueForcePair(VNI, false)));
+ Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), FP));
- // This was the first time (RegIdx, ParentVNI) was mapped.
- // Keep it as a simple def without any liveness.
- if (InsP.second)
+ // This was the first time (RegIdx, ParentVNI) was mapped, and it is not
+ // forced. Keep it as a simple def without any liveness.
+ if (!Force && InsP.second)
return VNI;
// If the previous value was a simple mapping, add liveness for it now.
if (VNInfo *OldVNI = InsP.first->second.getPointer()) {
- SlotIndex Def = OldVNI->def;
- LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), OldVNI));
- // No longer a simple mapping. Switch to a complex, non-forced mapping.
- InsP.first->second = ValueForcePair();
+ addDeadDef(*LI, OldVNI, Original);
+
+ // No longer a simple mapping. Switch to a complex mapping. If the
+ // interval has subranges, make it a forced mapping.
+ InsP.first->second = ValueForcePair(nullptr, Force);
}
// This is a complex mapping, add liveness for VNI
- SlotIndex Def = VNI->def;
- LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI));
-
+ addDeadDef(*LI, VNI, Original);
return VNI;
}
@@ -431,9 +481,8 @@ void SplitEditor::forceRecompute(unsigne
// This was previously a single mapping. Make sure the old def is represented
// by a trivial live range.
- SlotIndex Def = VNI->def;
- LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
- LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI));
+ addDeadDef(LIS.getInterval(Edit->get(RegIdx)), VNI, false);
+
// Mark as complex mapped, forced.
VFP = ValueForcePair(nullptr, true);
}
@@ -455,13 +504,18 @@ VNInfo *SplitEditor::defFromParent(unsig
unsigned Original = VRM.getOriginal(Edit->get(RegIdx));
LiveInterval &OrigLI = LIS.getInterval(Original);
VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);
- LiveRangeEdit::Remat RM(ParentVNI);
- RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
- if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
- Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late);
- ++NumRemats;
- } else {
+ bool DidRemat = false;
+ if (OrigVNI) {
+ LiveRangeEdit::Remat RM(ParentVNI);
+ RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
+ if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
+ Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late);
+ ++NumRemats;
+ DidRemat = true;
+ }
+ }
+ if (!DidRemat) {
// Can't remat, just insert a copy from parent.
CopyMI = BuildMI(MBB, I, DebugLoc(), TII.get(TargetOpcode::COPY), LI->reg)
.addReg(Edit->getReg());
@@ -472,7 +526,7 @@ VNInfo *SplitEditor::defFromParent(unsig
}
// Define the value in Reg.
- return defValue(RegIdx, ParentVNI, Def);
+ return defValue(RegIdx, ParentVNI, Def, false);
}
/// Create a new virtual register and live interval.
@@ -944,14 +998,15 @@ bool SplitEditor::transferValues() {
}
// The interval [Start;End) is continuously mapped to RegIdx, ParentVNI.
- DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx);
- LiveRange &LR = LIS.getInterval(Edit->get(RegIdx));
+ DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx
+ << '(' << PrintReg(Edit->get(RegIdx)) << ')');
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
// Check for a simply defined value that can be blitted directly.
ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id));
if (VNInfo *VNI = VFP.getPointer()) {
DEBUG(dbgs() << ':' << VNI->id);
- LR.addSegment(LiveInterval::Segment(Start, End, VNI));
+ LI.addSegment(LiveInterval::Segment(Start, End, VNI));
Start = End;
continue;
}
@@ -975,7 +1030,7 @@ bool SplitEditor::transferValues() {
// The first block may be live-in, or it may have its own def.
if (Start != BlockStart) {
- VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End));
+ VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
assert(VNI && "Missing def for complex mapped value");
DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber());
// MBB has its own def. Is it also live-out?
@@ -995,7 +1050,7 @@ bool SplitEditor::transferValues() {
if (BlockStart == ParentVNI->def) {
// This block has the def of a parent PHI, so it isn't live-in.
assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?");
- VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End));
+ VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
assert(VNI && "Missing def for complex mapped parent PHI");
if (End >= BlockEnd)
LRC.setLiveOutValue(&*MBB, VNI); // Live-out as well.
@@ -1003,10 +1058,10 @@ bool SplitEditor::transferValues() {
// This block needs a live-in value. The last block covered may not
// be live-out.
if (End < BlockEnd)
- LRC.addLiveInBlock(LR, MDT[&*MBB], End);
+ LRC.addLiveInBlock(LI, MDT[&*MBB], End);
else {
// Live-through, and we don't know the value.
- LRC.addLiveInBlock(LR, MDT[&*MBB]);
+ LRC.addLiveInBlock(LI, MDT[&*MBB]);
LRC.setLiveOutValue(&*MBB, nullptr);
}
}
@@ -1025,42 +1080,84 @@ bool SplitEditor::transferValues() {
return Skipped;
}
+static bool removeDeadSegment(SlotIndex Def, LiveRange &LR) {
+ const LiveRange::Segment *Seg = LR.getSegmentContaining(Def);
+ if (Seg == nullptr)
+ return true;
+ if (Seg->end != Def.getDeadSlot())
+ return false;
+ // This is a dead PHI. Remove it.
+ LR.removeSegment(*Seg, true);
+ return true;
+}
+
+void SplitEditor::extendPHIRange(MachineBasicBlock &B, LiveRangeCalc &LRC,
+ LiveRange &LR, ArrayRef<SlotIndex> Undefs) {
+ for (MachineBasicBlock *P : B.predecessors()) {
+ SlotIndex End = LIS.getMBBEndIdx(P);
+ SlotIndex LastUse = End.getPrevSlot();
+ // The predecessor may not have a live-out value. That is OK, like an
+ // undef PHI operand.
+ if (Edit->getParent().liveAt(LastUse))
+ LRC.extend(LR, End, /*PhysReg=*/0, Undefs);
+ }
+}
+
void SplitEditor::extendPHIKillRanges() {
// Extend live ranges to be live-out for successor PHI values.
- for (const VNInfo *PHIVNI : Edit->getParent().valnos) {
- if (PHIVNI->isUnused() || !PHIVNI->isPHIDef())
- continue;
- unsigned RegIdx = RegAssign.lookup(PHIVNI->def);
- LiveRange &LR = LIS.getInterval(Edit->get(RegIdx));
- // Check whether PHI is dead.
- const LiveRange::Segment *Segment = LR.getSegmentContaining(PHIVNI->def);
- assert(Segment != nullptr && "Missing segment for VNI");
- if (Segment->end == PHIVNI->def.getDeadSlot()) {
- // This is a dead PHI. Remove it.
- LR.removeSegment(*Segment, true);
+ // Visit each PHI def slot in the parent live interval. If the def is dead,
+ // remove it. Otherwise, extend the live interval to reach the end indexes
+ // of all predecessor blocks.
+
+ LiveInterval &ParentLI = Edit->getParent();
+ for (const VNInfo *V : ParentLI.valnos) {
+ if (V->isUnused() || !V->isPHIDef())
continue;
- }
+ unsigned RegIdx = RegAssign.lookup(V->def);
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
LiveRangeCalc &LRC = getLRCalc(RegIdx);
- MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def);
- for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
- PE = MBB->pred_end(); PI != PE; ++PI) {
- SlotIndex End = LIS.getMBBEndIdx(*PI);
- SlotIndex LastUse = End.getPrevSlot();
- // The predecessor may not have a live-out value. That is OK, like an
- // undef PHI operand.
- if (Edit->getParent().liveAt(LastUse)) {
- assert(RegAssign.lookup(LastUse) == RegIdx &&
- "Different register assignment in phi predecessor");
- LRC.extend(LR, End);
- }
+ MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
+ if (!removeDeadSegment(V->def, LI))
+ extendPHIRange(B, LRC, LI, /*Undefs=*/{});
+ }
+
+ SmallVector<SlotIndex, 4> Undefs;
+ LiveRangeCalc SubLRC;
+
+ for (LiveInterval::SubRange &PS : ParentLI.subranges()) {
+ for (const VNInfo *V : PS.valnos) {
+ if (V->isUnused() || !V->isPHIDef())
+ continue;
+ unsigned RegIdx = RegAssign.lookup(V->def);
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
+ LiveInterval::SubRange &S = getSubRangeForMask(PS.LaneMask, LI);
+ if (removeDeadSegment(V->def, S))
+ continue;
+
+ MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
+ SubLRC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
+ &LIS.getVNInfoAllocator());
+ Undefs.clear();
+ LI.computeSubRangeUndefs(Undefs, PS.LaneMask, MRI, *LIS.getSlotIndexes());
+ extendPHIRange(B, SubLRC, S, Undefs);
}
}
}
/// rewriteAssigned - Rewrite all uses of Edit->getReg().
void SplitEditor::rewriteAssigned(bool ExtendRanges) {
+ struct ExtPoint {
+ ExtPoint(const MachineOperand &O, unsigned R, SlotIndex N)
+ : MO(O), RegIdx(R), Next(N) {}
+ MachineOperand MO;
+ unsigned RegIdx;
+ SlotIndex Next;
+ };
+
+ SmallVector<ExtPoint,4> ExtPoints;
+
for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()),
RE = MRI.reg_end(); RI != RE;) {
MachineOperand &MO = *RI;
@@ -1082,8 +1179,8 @@ void SplitEditor::rewriteAssigned(bool E
// Rewrite to the mapped register at Idx.
unsigned RegIdx = RegAssign.lookup(Idx);
- LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
- MO.setReg(LI->reg);
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
+ MO.setReg(LI.reg);
DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t'
<< Idx << ':' << RegIdx << '\t' << *MI);
@@ -1095,7 +1192,7 @@ void SplitEditor::rewriteAssigned(bool E
if (MO.isDef()) {
if (!MO.getSubReg() && !MO.isEarlyClobber())
continue;
- // We may wan't to extend a live range for a partial redef, or for a use
+ // We may want to extend a live range for a partial redef, or for a use
// tied to an early clobber.
Idx = Idx.getPrevSlot();
if (!Edit->getParent().liveAt(Idx))
@@ -1103,7 +1200,56 @@ void SplitEditor::rewriteAssigned(bool E
} else
Idx = Idx.getRegSlot(true);
- getLRCalc(RegIdx).extend(*LI, Idx.getNextSlot());
+ SlotIndex Next = Idx.getNextSlot();
+ if (LI.hasSubRanges()) {
+ // We have to delay extending subranges until we have seen all operands
+ // defining the register. This is because a <def,read-undef> operand
+ // will create an "undef" point, and we cannot extend any subranges
+ // until all of them have been accounted for.
+ ExtPoints.push_back(ExtPoint(MO, RegIdx, Next));
+ } else {
+ LiveRangeCalc &LRC = getLRCalc(RegIdx);
+ LRC.extend(LI, Next, 0, ArrayRef<SlotIndex>());
+ }
+ }
+
+ for (ExtPoint &EP : ExtPoints) {
+ LiveInterval &LI = LIS.getInterval(Edit->get(EP.RegIdx));
+ assert(LI.hasSubRanges());
+
+ LiveRangeCalc SubLRC;
+ unsigned Reg = EP.MO.getReg(), Sub = EP.MO.getSubReg();
+ LaneBitmask LM = Sub != 0 ? TRI.getSubRegIndexLaneMask(Sub)
+ : MRI.getMaxLaneMaskForVReg(Reg);
+ // If this is a non-read-undef definition of a sub-register, extend
+ // subranges for everything except that sub-register.
+ if (Sub != 0 && EP.MO.isDef())
+ LM = MRI.getMaxLaneMaskForVReg(Reg) & ~LM;
+ for (LiveInterval::SubRange &S : LI.subranges()) {
+ if (!(S.LaneMask & LM))
+ continue;
+ // The problem here can be that the new register may have been created
+ // for a partially defined original register. For example:
+ // %vreg827:subreg_hireg<def,read-undef> = ...
+ // ...
+ // %vreg828<def> = COPY %vreg827
+ if (S.empty())
+ continue;
+ SubLRC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
+ &LIS.getVNInfoAllocator());
+ SmallVector<SlotIndex, 4> Undefs;
+ LI.computeSubRangeUndefs(Undefs, S.LaneMask, MRI, *LIS.getSlotIndexes());
+ SubLRC.extend(S, EP.Next, 0, Undefs);
+ }
+ }
+
+ for (unsigned R : *Edit) {
+ LiveInterval &LI = LIS.getInterval(R);
+ if (!LI.hasSubRanges())
+ continue;
+ LI.clear();
+ LI.removeEmptySubRanges();
+ LIS.constructMainRangeFromSubranges(LI);
}
}
@@ -1146,7 +1292,7 @@ void SplitEditor::finish(SmallVectorImpl
if (ParentVNI->isUnused())
continue;
unsigned RegIdx = RegAssign.lookup(ParentVNI->def);
- defValue(RegIdx, ParentVNI, ParentVNI->def);
+ defValue(RegIdx, ParentVNI, ParentVNI->def, true);
// Force rematted values to be recomputed everywhere.
// The new live ranges may be truncated.
@@ -1182,8 +1328,9 @@ void SplitEditor::finish(SmallVectorImpl
deleteRematVictims();
// Get rid of unused values and set phi-kill flags.
- for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I) {
- LiveInterval &LI = LIS.getInterval(*I);
+ for (unsigned Reg : *Edit) {
+ LiveInterval &LI = LIS.getInterval(Reg);
+ LI.removeEmptySubRanges();
LI.RenumberValues();
}
Modified: llvm/trunk/lib/CodeGen/SplitKit.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SplitKit.h?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SplitKit.h (original)
+++ llvm/trunk/lib/CodeGen/SplitKit.h Wed Aug 24 08:37:55 2016
@@ -325,12 +325,30 @@ private:
return LRCalc[SpillMode != SM_Partition && RegIdx != 0];
}
+ /// Find a subrange corresponding to the lane mask @p LM in the live
+ /// interval @p LI. The interval @p LI is assumed to contain such a subrange.
+ /// This function is used to find corresponding subranges between the
+ /// original interval and the new intervals.
+ LiveInterval::SubRange &getSubRangeForMask(LaneBitmask LM, LiveInterval &LI);
+
+ /// Add a segment to the interval LI for the value number VNI. If LI has
+ /// subranges, corresponding segments will be added to them as well, but
+ /// with newly created value numbers. If Original is true, dead def will
+ /// only be added a subrange of LI if the corresponding subrange of the
+ /// original interval has a def at this index. Otherwise, all subranges
+ /// of LI will be updated.
+ void addDeadDef(LiveInterval &LI, VNInfo *VNI, bool Original);
+
/// defValue - define a value in RegIdx from ParentVNI at Idx.
/// Idx does not have to be ParentVNI->def, but it must be contained within
/// ParentVNI's live range in ParentLI. The new value is added to the value
- /// map.
+ /// map. The value being defined may either come from rematerialization
+ /// (or an inserted copy), or it may be coming from the original interval.
+ /// The parameter Original should be true in the latter case, otherwise
+ /// it should be false.
/// Return the new LI value.
- VNInfo *defValue(unsigned RegIdx, const VNInfo *ParentVNI, SlotIndex Idx);
+ VNInfo *defValue(unsigned RegIdx, const VNInfo *ParentVNI, SlotIndex Idx,
+ bool Original);
/// forceRecompute - Force the live range of ParentVNI in RegIdx to be
/// recomputed by LiveRangeCalc::extend regardless of the number of defs.
@@ -368,6 +386,11 @@ private:
/// Return true if any ranges were skipped.
bool transferValues();
+ /// Live range @p LR has a live PHI def at the beginning of block @p B.
+ /// Extend the range @p LR of all predecessor values that reach this def.
+ void extendPHIRange(MachineBasicBlock &B, LiveRangeCalc &LRC,
+ LiveRange &LR, ArrayRef<SlotIndex>);
+
/// extendPHIKillRanges - Extend the ranges of all values killed by original
/// parent PHIDefs.
void extendPHIKillRanges();
Modified: llvm/trunk/lib/CodeGen/VirtRegMap.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/VirtRegMap.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/VirtRegMap.cpp (original)
+++ llvm/trunk/lib/CodeGen/VirtRegMap.cpp Wed Aug 24 08:37:55 2016
@@ -338,6 +338,7 @@ bool VirtRegRewriter::readsUndefSubreg(c
assert(LI.liveAt(BaseIndex) &&
"Reads of completely dead register should be marked undef already");
unsigned SubRegIdx = MO.getSubReg();
+ assert(SubRegIdx != 0 && LI.hasSubRanges());
LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);
// See if any of the relevant subregister liveranges is defined at this point.
for (const LiveInterval::SubRange &SR : LI.subranges()) {
Modified: llvm/trunk/lib/Target/Hexagon/HexagonExpandCondsets.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Hexagon/HexagonExpandCondsets.cpp?rev=279625&r1=279624&r2=279625&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Hexagon/HexagonExpandCondsets.cpp (original)
+++ llvm/trunk/lib/Target/Hexagon/HexagonExpandCondsets.cpp Wed Aug 24 08:37:55 2016
@@ -459,9 +459,15 @@ void HexagonExpandCondsets::updateDeadsI
if (HII->isPredicated(*DefI))
PredDefs.push_back(Seg.start);
}
+
+ SmallVector<SlotIndex,8> Undefs;
+ LiveInterval &LI = LIS->getInterval(Reg);
+ LI.computeSubRangeUndefs(Undefs, LM, *MRI, *LIS->getSlotIndexes());
+
for (auto &SI : PredDefs) {
MachineBasicBlock *BB = LIS->getMBBFromIndex(SI);
- if (Range.extendInBlock(LIS->getMBBStartIdx(BB), SI))
+ auto P = Range.extendInBlock(Undefs, LIS->getMBBStartIdx(BB), SI);
+ if (P.first != nullptr || P.second)
SI = SlotIndex();
}
Added: llvm/trunk/test/CodeGen/AMDGPU/subreg-intervals.mir
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/subreg-intervals.mir?rev=279625&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/AMDGPU/subreg-intervals.mir (added)
+++ llvm/trunk/test/CodeGen/AMDGPU/subreg-intervals.mir Wed Aug 24 08:37:55 2016
@@ -0,0 +1,51 @@
+# RUN: llc -march=amdgcn -run-pass liveintervals -debug-only=regalloc -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# CHECK: INTERVALS
+# CHECK: vreg0
+# CHECK-LABEL: Machine code for function test0:
+
+# CHECK: INTERVALS
+# CHECK: vreg0
+# CHECK-LABEL: Machine code for function test1:
+
+--- |
+ define void @test0() { ret void }
+ define void @test1() { ret void }
+...
+---
+name: test0
+registers:
+ - { id: 0, class: sreg_64 }
+body: |
+ bb.0:
+ S_NOP 0, implicit-def %0
+ S_NOP 0, implicit %0
+
+ S_NOP 0, implicit-def undef %0.sub0
+ S_NOP 0, implicit %0
+...
+---
+name: test1
+registers:
+ - { id: 0, class: sreg_64 }
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc
+ S_BRANCH %bb.2
+
+ bb.1:
+ successors: %bb.3
+ S_NOP 0, implicit-def undef %0.sub0
+ S_BRANCH %bb.3
+
+ bb.2:
+ successors: %bb.3
+ S_NOP 0, implicit-def %0
+ S_BRANCH %bb.3
+
+ bb.3:
+ S_NOP 0
+ S_NOP 0, implicit %0
+...
Added: llvm/trunk/test/CodeGen/Hexagon/regalloc-bad-undef.mir
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/Hexagon/regalloc-bad-undef.mir?rev=279625&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/Hexagon/regalloc-bad-undef.mir (added)
+++ llvm/trunk/test/CodeGen/Hexagon/regalloc-bad-undef.mir Wed Aug 24 08:37:55 2016
@@ -0,0 +1,205 @@
+# RUN: llc -march=hexagon -hexagon-subreg-liveness -start-after machine-scheduler -stop-after stack-slot-coloring -o - %s | FileCheck %s
+
+--- |
+ target triple = "hexagon"
+
+ ; Function Attrs: nounwind optsize
+ define void @main() #0 {
+ entry:
+ br label %for.body
+
+ for.body: ; preds = %if.end82, %entry
+ %lsr.iv = phi i32 [ %lsr.iv.next, %if.end82 ], [ 524288, %entry ]
+ %call9 = tail call i32 @lrand48() #0
+ %conv10 = sext i32 %call9 to i64
+ %shr11 = lshr i64 %conv10, 9
+ %or12 = or i64 0, %shr11
+ %call14 = tail call i32 @lrand48() #0
+ %conv15138 = zext i32 %call14 to i64
+ %shr16 = lshr i64 %conv15138, 9
+ %0 = call i64 @llvm.hexagon.S2.extractup(i64 %conv15138, i32 22, i32 9)
+ %1 = shl i64 %0, 42
+ %shl17 = shl i64 %shr16, 42
+ %or22 = or i64 0, %1
+ %or26 = or i64 %or22, 0
+ %shr30 = lshr i64 undef, 25
+ %2 = call i64 @llvm.hexagon.S2.extractup(i64 undef, i32 6, i32 25)
+ %and = and i64 %shr30, 63
+ %sub = shl i64 2, %2
+ %add = add i64 %sub, -1
+ %shl37 = shl i64 %add, 0
+ %call38 = tail call i32 @lrand48() #0
+ %conv39141 = zext i32 %call38 to i64
+ %shr40 = lshr i64 %conv39141, 25
+ %3 = call i64 @llvm.hexagon.S2.extractup(i64 %conv39141, i32 6, i32 25)
+ %and41 = and i64 %shr40, 63
+ %sub43 = shl i64 2, %3
+ %add45 = add i64 %sub43, -1
+ %call46 = tail call i32 @lrand48() #0
+ %shr48 = lshr i64 undef, 25
+ %4 = call i64 @llvm.hexagon.S2.extractup(i64 undef, i32 6, i32 25)
+ %and49 = and i64 %shr48, 63
+ %shl50 = shl i64 %add45, %4
+ %and52 = and i64 %shl37, %or12
+ %and54 = and i64 %shl50, %or26
+ store i64 %and54, i64* undef, align 8
+ %cmp56 = icmp eq i64 %and52, 0
+ br i1 %cmp56, label %for.end, label %if.end82
+
+ if.end82: ; preds = %for.body
+ %lsr.iv.next = add nsw i32 %lsr.iv, -1
+ %exitcond = icmp eq i32 %lsr.iv.next, 0
+ br i1 %exitcond, label %for.end, label %for.body
+
+ for.end: ; preds = %if.end82, %for.body
+ unreachable
+ }
+
+ declare i32 @lrand48() #0
+ declare i64 @llvm.hexagon.S2.extractup(i64, i32, i32) #1
+
+ attributes #0 = { nounwind optsize "target-cpu"="hexagonv55" "target-features"="-hvx,-hvx-double" }
+ attributes #1 = { nounwind readnone }
+
+...
+---
+name: main
+alignment: 2
+tracksRegLiveness: true
+tracksSubRegLiveness: true
+registers:
+ - { id: 0, class: intregs }
+ - { id: 1, class: intregs }
+ - { id: 2, class: intregs }
+ - { id: 3, class: intregs }
+ - { id: 4, class: doubleregs }
+ - { id: 5, class: intregs }
+ - { id: 6, class: doubleregs }
+ - { id: 7, class: doubleregs }
+ - { id: 8, class: doubleregs }
+ - { id: 9, class: doubleregs }
+ - { id: 10, class: intregs }
+ - { id: 11, class: doubleregs }
+ - { id: 12, class: doubleregs }
+ - { id: 13, class: doubleregs }
+ - { id: 14, class: intregs }
+ - { id: 15, class: doubleregs }
+ - { id: 16, class: doubleregs }
+ - { id: 17, class: intregs }
+ - { id: 18, class: doubleregs }
+ - { id: 19, class: intregs }
+ - { id: 20, class: doubleregs }
+ - { id: 21, class: doubleregs }
+ - { id: 22, class: doubleregs }
+ - { id: 23, class: intregs }
+ - { id: 24, class: doubleregs }
+ - { id: 25, class: predregs }
+ - { id: 26, class: predregs }
+ - { id: 27, class: intregs }
+ - { id: 28, class: intregs }
+ - { id: 29, class: doubleregs }
+ - { id: 30, class: intregs }
+ - { id: 31, class: intregs }
+ - { id: 32, class: doubleregs }
+ - { id: 33, class: intregs }
+ - { id: 34, class: intregs }
+ - { id: 35, class: doubleregs }
+ - { id: 36, class: doubleregs }
+ - { id: 37, class: intregs }
+ - { id: 38, class: intregs }
+ - { id: 39, class: doubleregs }
+ - { id: 40, class: doubleregs }
+ - { id: 41, class: intregs }
+ - { id: 42, class: intregs }
+ - { id: 43, class: doubleregs }
+ - { id: 44, class: intregs }
+ - { id: 45, class: intregs }
+ - { id: 46, class: doubleregs }
+ - { id: 47, class: doubleregs }
+ - { id: 48, class: doubleregs }
+ - { id: 49, class: doubleregs }
+ - { id: 50, class: doubleregs }
+ - { id: 51, class: doubleregs }
+ - { id: 52, class: intregs }
+ - { id: 53, class: intregs }
+ - { id: 54, class: intregs }
+ - { id: 55, class: doubleregs }
+ - { id: 56, class: doubleregs }
+ - { id: 57, class: intregs }
+ - { id: 58, class: intregs }
+ - { id: 59, class: intregs }
+frameInfo:
+ isFrameAddressTaken: false
+ isReturnAddressTaken: false
+ hasStackMap: false
+ hasPatchPoint: false
+ stackSize: 0
+ offsetAdjustment: 0
+ maxAlignment: 0
+ adjustsStack: false
+ hasCalls: true
+ maxCallFrameSize: 0
+ hasOpaqueSPAdjustment: false
+ hasVAStart: false
+ hasMustTailInVarArgFunc: false
+body: |
+ bb.0.entry:
+ successors: %bb.1.for.body
+
+ %59 = A2_tfrsi 524288
+ undef %32.subreg_hireg = A2_tfrsi 0
+ %8 = S2_extractup undef %9, 6, 25
+ %47 = A2_tfrpi 2
+ %13 = A2_tfrpi -1
+ %13 = S2_asl_r_p_acc %13, %47, %8.subreg_loreg
+ %51 = A2_tfrpi 0
+
+ ; CHECK: %d2 = S2_extractup undef %d0, 6, 25
+ ; CHECK: %d0 = A2_tfrpi 2
+ ; CHECK: %d13 = A2_tfrpi -1
+ ; CHECK-NOT: undef %r4
+
+ bb.1.for.body:
+ successors: %bb.3.for.end, %bb.2.if.end82
+
+ ADJCALLSTACKDOWN 0, implicit-def dead %r29, implicit-def dead %r30, implicit %r31, implicit %r30, implicit %r29
+ J2_call @lrand48, implicit-def dead %d0, implicit-def dead %d1, implicit-def dead %d2, implicit-def dead %d3, implicit-def dead %d4, implicit-def dead %d5, implicit-def dead %d6, implicit-def dead %d7, implicit-def dead %r28, implicit-def dead %r31, implicit-def dead %p0, implicit-def dead %p1, implicit-def dead %p2, implicit-def dead %p3, implicit-def dead %m0, implicit-def dead %m1, implicit-def dead %lc0, implicit-def dead %lc1, implicit-def dead %sa0, implicit-def dead %sa1, implicit-def dead %usr, implicit-def %usr_ovf, implicit-def dead %cs0, implicit-def dead %cs1, implicit-def dead %w0, implicit-def dead %w1, implicit-def dead %w2, implicit-def dead %w3, implicit-def dead %w4, implicit-def dead %w5, implicit-def dead %w6, implicit-def dead %w7, implicit-def dead %w8, implicit-def dead %w9, implicit-def dead %w10, implicit-def dead %w11, implicit-def dead %w12, implicit-def dead %w13, implicit-def dead %w14, implicit-def dead %w15, implicit-def dead %q0, implicit-def dead
%q1, implicit-def dead %q2, implicit-def dead %q3, implicit-def %r0
+ ADJCALLSTACKUP 0, 0, implicit-def dead %r29, implicit-def dead %r30, implicit-def dead %r31, implicit %r29
+ undef %29.subreg_loreg = COPY killed %r0
+ %29.subreg_hireg = S2_asr_i_r %29.subreg_loreg, 31
+ ADJCALLSTACKDOWN 0, implicit-def dead %r29, implicit-def dead %r30, implicit %r31, implicit %r30, implicit %r29
+ J2_call @lrand48, implicit-def dead %d0, implicit-def dead %d1, implicit-def dead %d2, implicit-def dead %d3, implicit-def dead %d4, implicit-def dead %d5, implicit-def dead %d6, implicit-def dead %d7, implicit-def dead %r28, implicit-def dead %r31, implicit-def dead %p0, implicit-def dead %p1, implicit-def dead %p2, implicit-def dead %p3, implicit-def dead %m0, implicit-def dead %m1, implicit-def dead %lc0, implicit-def dead %lc1, implicit-def dead %sa0, implicit-def dead %sa1, implicit-def dead %usr, implicit-def %usr_ovf, implicit-def dead %cs0, implicit-def dead %cs1, implicit-def dead %w0, implicit-def dead %w1, implicit-def dead %w2, implicit-def dead %w3, implicit-def dead %w4, implicit-def dead %w5, implicit-def dead %w6, implicit-def dead %w7, implicit-def dead %w8, implicit-def dead %w9, implicit-def dead %w10, implicit-def dead %w11, implicit-def dead %w12, implicit-def dead %w13, implicit-def dead %w14, implicit-def dead %w15, implicit-def dead %q0, implicit-def dead
%q1, implicit-def dead %q2, implicit-def dead %q3, implicit-def %r0
+ ADJCALLSTACKUP 0, 0, implicit-def dead %r29, implicit-def dead %r30, implicit-def dead %r31, implicit %r29
+ %32.subreg_loreg = COPY killed %r0
+ %7 = S2_extractup %32, 22, 9
+ ADJCALLSTACKDOWN 0, implicit-def dead %r29, implicit-def dead %r30, implicit %r31, implicit %r30, implicit %r29
+ J2_call @lrand48, implicit-def dead %d0, implicit-def dead %d1, implicit-def dead %d2, implicit-def dead %d3, implicit-def dead %d4, implicit-def dead %d5, implicit-def dead %d6, implicit-def dead %d7, implicit-def dead %r28, implicit-def dead %r31, implicit-def dead %p0, implicit-def dead %p1, implicit-def dead %p2, implicit-def dead %p3, implicit-def dead %m0, implicit-def dead %m1, implicit-def dead %lc0, implicit-def dead %lc1, implicit-def dead %sa0, implicit-def dead %sa1, implicit-def dead %usr, implicit-def %usr_ovf, implicit-def dead %cs0, implicit-def dead %cs1, implicit-def dead %w0, implicit-def dead %w1, implicit-def dead %w2, implicit-def dead %w3, implicit-def dead %w4, implicit-def dead %w5, implicit-def dead %w6, implicit-def dead %w7, implicit-def dead %w8, implicit-def dead %w9, implicit-def dead %w10, implicit-def dead %w11, implicit-def dead %w12, implicit-def dead %w13, implicit-def dead %w14, implicit-def dead %w15, implicit-def dead %q0, implicit-def dead
%q1, implicit-def dead %q2, implicit-def dead %q3, implicit-def %r0
+ ADJCALLSTACKUP 0, 0, implicit-def dead %r29, implicit-def dead %r30, implicit-def dead %r31, implicit %r29
+ undef %43.subreg_loreg = COPY killed %r0
+ %43.subreg_hireg = COPY %32.subreg_hireg
+ %16 = S2_extractup %43, 6, 25
+ %18 = A2_tfrpi -1
+ %18 = S2_asl_r_p_acc %18, %47, %16.subreg_loreg
+ ADJCALLSTACKDOWN 0, implicit-def dead %r29, implicit-def dead %r30, implicit %r31, implicit %r30, implicit %r29
+ J2_call @lrand48, implicit-def dead %d0, implicit-def dead %d1, implicit-def dead %d2, implicit-def dead %d3, implicit-def dead %d4, implicit-def dead %d5, implicit-def dead %d6, implicit-def dead %d7, implicit-def dead %r28, implicit-def dead %r31, implicit-def dead %p0, implicit-def dead %p1, implicit-def dead %p2, implicit-def dead %p3, implicit-def dead %m0, implicit-def dead %m1, implicit-def dead %lc0, implicit-def dead %lc1, implicit-def dead %sa0, implicit-def dead %sa1, implicit-def dead %usr, implicit-def %usr_ovf, implicit-def dead %cs0, implicit-def dead %cs1, implicit-def dead %w0, implicit-def dead %w1, implicit-def dead %w2, implicit-def dead %w3, implicit-def dead %w4, implicit-def dead %w5, implicit-def dead %w6, implicit-def dead %w7, implicit-def dead %w8, implicit-def dead %w9, implicit-def dead %w10, implicit-def dead %w11, implicit-def dead %w12, implicit-def dead %w13, implicit-def dead %w14, implicit-def dead %w15, implicit-def dead %q0, implicit-def dead
%q1, implicit-def dead %q2, implicit-def dead %q3
+ ADJCALLSTACKUP 0, 0, implicit-def dead %r29, implicit-def dead %r30, implicit-def dead %r31, implicit %r29
+ %22 = S2_asl_r_p %18, %8.subreg_loreg
+ %21 = COPY %13
+ %21 = S2_lsr_i_p_and %21, %29, 9
+ %22 = S2_asl_i_p_and %22, %7, 42
+ S2_storerd_io undef %23, 0, %22 :: (store 8 into `i64* undef`)
+ %25 = C2_cmpeqp %21, %51
+ J2_jumpt %25, %bb.3.for.end, implicit-def dead %pc
+ J2_jump %bb.2.if.end82, implicit-def dead %pc
+
+ bb.2.if.end82:
+ successors: %bb.3.for.end, %bb.1.for.body
+
+ %59 = A2_addi %59, -1
+ %26 = C2_cmpeqi %59, 0
+ J2_jumpf %26, %bb.1.for.body, implicit-def dead %pc
+ J2_jump %bb.3.for.end, implicit-def dead %pc
+
+ bb.3.for.end:
+
+...
More information about the llvm-commits
mailing list