[llvm-branch-commits] [llvm] CodeGen: Simplify def segment extension in shrinkToUses (PR #242869)
Matt Arsenault via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 15 13:25:04 PST 2026
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/242869
During extendSegmentsToUses, Segments holds only one def segment per
value, and none extend past their block. Live-in segments are kept
separately in LiveIns. To extend a def to a use, find the last segment
starting before the use, check that it starts in the use's block, and
move its end to the use. This replaces extendInBlock, whose general
search and merging of overlapped segments never applies here.
Also combine the per-block live-out and live-in tables into one struct.
llc instructions:u:
CTMark x86_64: -0.07%
CTMark arm64: -0.10%
AMDGPU sample: -0.27%
Largest AMDGPU improvements:
rocprim warp_sort (gfx1100): -1.89%
rocsolver stedc (gfx942): -1.70%
rocsolver stedcx (gfx942): -1.64%
rocsolver trtri-specialized-kernels-d (gfx942): -1.46%
rocsolver potf2-specialized-kernels-d (gfx942): -1.28%
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
>From 8273441df4123c6e731ac8d871e84bd1d7acd33b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 15 Nov 2026 13:07:01 -0800
Subject: [PATCH] CodeGen: Simplify def segment extension in shrinkToUses
During extendSegmentsToUses, Segments holds only one def segment per
value, and none extend past their block. Live-in segments are kept
separately in LiveIns. To extend a def to a use, find the last segment
starting before the use, check that it starts in the use's block, and
move its end to the use. This replaces extendInBlock, whose general
search and merging of overlapped segments never applies here.
Also combine the per-block live-out and live-in tables into one struct.
llc instructions:u:
CTMark x86_64: -0.07%
CTMark arm64: -0.10%
AMDGPU sample: -0.27%
Largest AMDGPU improvements:
rocprim warp_sort (gfx1100): -1.89%
rocsolver stedc (gfx942): -1.70%
rocsolver stedcx (gfx942): -1.64%
rocsolver trtri-specialized-kernels-d (gfx942): -1.46%
rocsolver potf2-specialized-kernels-d (gfx942): -1.28%
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/LiveIntervals.h | 24 +++++++----
llvm/lib/CodeGen/LiveIntervals.cpp | 51 +++++++++++------------
2 files changed, 40 insertions(+), 35 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/LiveIntervals.h b/llvm/include/llvm/CodeGen/LiveIntervals.h
index eaf2a6cc7bd7e..39225e8a0fe2f 100644
--- a/llvm/include/llvm/CodeGen/LiveIntervals.h
+++ b/llvm/include/llvm/CodeGen/LiveIntervals.h
@@ -74,17 +74,23 @@ class LiveIntervals {
/// Live interval pointers for all the virtual registers.
IndexedMap<LiveInterval *, VirtReg2IndexFunctor> VirtRegIntervals;
- /// Indexed by block number. A block has been queued as live-out by the
- /// current extendSegmentsToUses call iff its entry equals LiveOutTag.
- SmallVector<unsigned, 0> LiveOutTags;
-
- /// Indexed by block number. Holds {tag, index into the pending live-in
- /// segments}; the index is only valid when the tag equals LiveOutTag.
- SmallVector<std::pair<unsigned, unsigned>, 0> LiveInSegments;
+ /// Per-block state for the current extendSegmentsToUses call. A tag is
+ /// current iff it equals ShrinkTag.
+ struct ShrinkBlockState {
+ /// Current if the block has been queued as live-out.
+ unsigned LiveOutTag = 0;
+ /// Current if LiveIn is valid.
+ unsigned LiveInTag = 0;
+ /// Index of the block's pending live-in segment.
+ unsigned LiveIn = 0;
+ };
+
+ /// Indexed by block number.
+ SmallVector<ShrinkBlockState, 0> ShrinkBlocks;
/// Tag for the current extendSegmentsToUses call, incremented on each call
- /// so stale LiveOutTags entries never need clearing.
- unsigned LiveOutTag = 0;
+ /// so stale ShrinkBlocks entries never need clearing.
+ unsigned ShrinkTag = 0;
/// Sorted list of instructions with register mask operands. Always use the
/// 'r' slot, RegMasks are normal clobbers, not early clobbers.
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index 0e5d87a1fff5e..912be97828cf5 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -151,8 +151,7 @@ void LiveIntervals::clear() {
RegMaskSlots.clear();
RegMaskBits.clear();
RegMaskBlocks.clear();
- LiveOutTags.clear();
- LiveInSegments.clear();
+ ShrinkBlocks.clear();
for (LiveRange *LR : RegUnitRanges) {
if (LR)
@@ -179,8 +178,7 @@ void LiveIntervals::analyze(MachineFunction &fn) {
// Allocate space for all virtual registers.
VirtRegIntervals.resize(MRI->getNumVirtRegs());
- LiveOutTags.assign(MF->getNumBlockIDs(), 0);
- LiveInSegments.assign(MF->getNumBlockIDs(), {0, 0});
+ ShrinkBlocks.assign(MF->getNumBlockIDs(), {});
computeVirtRegs();
computeRegMasks();
@@ -365,8 +363,7 @@ void LiveIntervals::insertMBBInMapsImpl(
assert(unsigned(MBB->getNumber()) == RegMaskBlocks.size() &&
"Blocks must be added in order.");
RegMaskBlocks.push_back(std::make_pair(RegMaskSlots.size(), 0));
- LiveOutTags.push_back(0);
- LiveInSegments.push_back({0, 0});
+ ShrinkBlocks.emplace_back();
}
//===----------------------------------------------------------------------===//
@@ -485,11 +482,10 @@ void LiveIntervals::extendSegmentsToUses(LiveRange &Segments,
Register Reg, LaneBitmask LaneMask) {
// Keep track of the PHIs that are in use.
SmallPtrSet<VNInfo*, 8> UsedPHIs;
- // Blocks that have already been added to WorkList as live-out.
- if (++LiveOutTag == 0) {
- llvm::fill(LiveOutTags, 0);
- LiveInSegments.assign(LiveInSegments.size(), {0, 0});
- LiveOutTag = 1;
+ // Start a new tag; entries from earlier calls are now stale.
+ if (++ShrinkTag == 0) {
+ ShrinkBlocks.assign(ShrinkBlocks.size(), {});
+ ShrinkTag = 1;
}
auto getSubRange = [](const LiveInterval &I, LaneBitmask M)
@@ -520,20 +516,23 @@ void LiveIntervals::extendSegmentsToUses(LiveRange &Segments,
const MachineBasicBlock *MBB = Indexes->getMBBFromIndex(Idx.getPrevSlot());
SlotIndex BlockStart = Indexes->getMBBStartIdx(MBB);
- // Extend the live range for VNI to be live at Idx.
- if (VNInfo *ExtVNI = Segments.extendInBlock(BlockStart, Idx)) {
- assert(ExtVNI == VNI && "Unexpected existing value number");
- (void)ExtVNI;
+ // Extend the live range for VNI to be live at Idx. Segments only holds def
+ // segments during the walk, and none extend past their block.
+ LiveRange::iterator Def = llvm::partition_point(
+ Segments, [Idx](const LiveRange::Segment &S) { return S.start < Idx; });
+ if (Def != Segments.begin() && (--Def)->start >= BlockStart) {
+ assert(Def->valno == VNI && "Unexpected existing value number");
+ Def->end = std::max(Def->end, Idx);
// Is this a PHIDef we haven't seen before?
if (!VNI->isPHIDef() || VNI->def != BlockStart ||
!UsedPHIs.insert(VNI).second)
continue;
// The PHI is live, make sure the predecessors are live-out.
for (const MachineBasicBlock *Pred : MBB->predecessors()) {
- unsigned &Tag = LiveOutTags[Pred->getNumber()];
- if (Tag == LiveOutTag)
+ unsigned &Tag = ShrinkBlocks[Pred->getNumber()].LiveOutTag;
+ if (Tag == ShrinkTag)
continue;
- Tag = LiveOutTag;
+ Tag = ShrinkTag;
SlotIndex Stop = Indexes->getMBBEndIdx(Pred);
// A predecessor is not required to have a live-out value for a PHI.
if (VNInfo *PVNI = OldRange.getVNInfoBefore(Stop))
@@ -542,16 +541,16 @@ void LiveIntervals::extendSegmentsToUses(LiveRange &Segments,
continue;
}
- auto &[InTag, InIdx] = LiveInSegments[MBB->getNumber()];
- if (InTag == LiveOutTag) {
- LiveRange::Segment &S = LiveIns[InIdx];
+ ShrinkBlockState &BS = ShrinkBlocks[MBB->getNumber()];
+ if (BS.LiveInTag == ShrinkTag) {
+ LiveRange::Segment &S = LiveIns[BS.LiveIn];
assert(S.valno == VNI && "Unexpected existing value number");
S.end = std::max(S.end, Idx);
continue;
}
- InTag = LiveOutTag;
- InIdx = LiveIns.size();
+ BS.LiveInTag = ShrinkTag;
+ BS.LiveIn = LiveIns.size();
// VNI is live-in to MBB.
LLVM_DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
@@ -559,10 +558,10 @@ void LiveIntervals::extendSegmentsToUses(LiveRange &Segments,
// Make sure VNI is live-out from the predecessors.
for (const MachineBasicBlock *Pred : MBB->predecessors()) {
- unsigned &Tag = LiveOutTags[Pred->getNumber()];
- if (Tag == LiveOutTag)
+ unsigned &Tag = ShrinkBlocks[Pred->getNumber()].LiveOutTag;
+ if (Tag == ShrinkTag)
continue;
- Tag = LiveOutTag;
+ Tag = ShrinkTag;
SlotIndex Stop = Indexes->getMBBEndIdx(Pred);
if (VNInfo *OldVNI = OldRange.getVNInfoBefore(Stop)) {
assert(OldVNI == VNI && "Wrong value out of predecessor");
More information about the llvm-branch-commits
mailing list