[llvm] b182b41 - [CodeGen] Correctly classify/mark dead defs when adjusting lane liveness (#215595)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 02:17:57 PDT 2026
Author: Lucas Ramirez
Date: 2026-09-08T11:17:49+02:00
New Revision: b182b415d9b2447cf1fd7231ebe28a96cfd1f9af
URL: https://github.com/llvm/llvm-project/commit/b182b415d9b2447cf1fd7231ebe28a96cfd1f9af
DIFF: https://github.com/llvm/llvm-project/commit/b182b415d9b2447cf1fd7231ebe28a96cfd1f9af.diff
LOG: [CodeGen] Correctly classify/mark dead defs when adjusting lane liveness (#215595)
Despite what the documentation of `adjustLaneLiveness` states, the
method never sets dead flags on dead def operands, even when missing
dead flags can later lead to machine verifier errors.
This makes the method identify dead definitions from definitions that
are initially thought to be alive, and makes it add a dead flag on the
last definition of a virtual register, matching the behavior expected by
the machine verifier (ref. "Instruction ending live segment on dead slot
has no dead flag").
`adjustLaneLiveness` and `detectDeadDefs` now also use the same
mechanism to identify dead definitions. It relies on comparing the
defined lanes of a definition with those that stay alive after it.
Added:
llvm/unittests/CodeGen/RegisterPressureTest.cpp
Modified:
llvm/include/llvm/CodeGen/RegisterPressure.h
llvm/lib/CodeGen/MachineScheduler.cpp
llvm/lib/CodeGen/RegisterPressure.cpp
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
llvm/unittests/CodeGen/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/RegisterPressure.h b/llvm/include/llvm/CodeGen/RegisterPressure.h
index 8cd9c8374357e..47c2a31d4317a 100644
--- a/llvm/include/llvm/CodeGen/RegisterPressure.h
+++ b/llvm/include/llvm/CodeGen/RegisterPressure.h
@@ -172,7 +172,10 @@ class RegisterOperands {
/// instruction which are not dead.
SmallVector<VRegMaskOrUnit, 8> Defs;
/// List of virtual registers and register units defined by the
- /// instruction but dead.
+ /// instruction but dead. Dead definitions should not necessarily be marked
+ /// with a dead flag. In this context a dead definition is just a definition
+ /// which doesn't define any register lane that remains live after the
+ /// defining instruction.
SmallVector<VRegMaskOrUnit, 8> DeadDefs;
/// Analyze the given instruction \p MI and fill in the Uses, Defs and
@@ -181,10 +184,11 @@ class RegisterOperands {
const MachineRegisterInfo &MRI, bool TrackLaneMasks,
bool IgnoreDead);
- /// Use liveness information to find dead defs not marked with a dead flag
- /// and move them to the DeadDefs vector.
- LLVM_ABI void detectDeadDefs(const MachineInstr &MI,
- const LiveIntervals &LIS);
+ /// Use liveness information to find dead defs at \p MI's dead slot not marked
+ /// with a dead flag and move them to the DeadDefs vector. This only considers
+ /// the merged live interval for defs, not the per-lane sub-ranges.
+ LLVM_ABI void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS,
+ const MachineRegisterInfo &MRI);
/// Use liveness information to find out which uses/defs are partially
/// undefined/dead at \p Pos and adjust the VRegMaskOrUnits accordingly.
@@ -200,9 +204,10 @@ class RegisterOperands {
MachineInstr &MI);
private:
- /// Adjusts the \p Def based on \p LiveAfterDef. The \p Def is removed from
- /// the Defs vector when no defined lane remains live after the def. Returns a
- /// pointer to the next definition to process in order in the Defs vector.
+ /// Adjusts the \p Def based on \p LiveAfterDef. The \p Def is moved from the
+ /// Defs vector to the DeadDefs vector when no defined lane remains live after
+ /// the def. Returns a pointer to the next definition to process in order in
+ /// the Defs vector.
VRegMaskOrUnit *adjustDef(VRegMaskOrUnit &Def, LaneBitmask LiveAfterDef);
/// Use liveness information at \p Pos to adjust the lanemask of all uses.
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 814ea3f8eeb05..960c75d8050b1 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1902,7 +1902,7 @@ void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) {
RegOpers.adjustLaneLiveness(*LIS, MRI, *MI);
} else {
// Adjust for missing dead-def flags.
- RegOpers.detectDeadDefs(*MI, *LIS);
+ RegOpers.detectDeadDefs(*MI, *LIS, MRI);
}
TopRPTracker.advance(RegOpers);
@@ -1936,7 +1936,7 @@ void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) {
RegOpers.adjustLaneLiveness(*LIS, MRI, *MI);
} else {
// Adjust for missing dead-def flags.
- RegOpers.detectDeadDefs(*MI, *LIS);
+ RegOpers.detectDeadDefs(*MI, *LIS, MRI);
}
if (BotRPTracker.getPos() != CurrentBottom)
diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp
index 799a825717bb9..c42dced2a5bea 100644
--- a/llvm/lib/CodeGen/RegisterPressure.cpp
+++ b/llvm/lib/CodeGen/RegisterPressure.cpp
@@ -232,13 +232,6 @@ void LiveRegSet::clear() {
Regs.clear();
}
-static const LiveRange *getLiveRange(const LiveIntervals &LIS,
- VirtRegOrUnit VRegOrUnit) {
- if (VRegOrUnit.isVirtualReg())
- return &LIS.getInterval(VRegOrUnit.asVirtualReg());
- return LIS.getCachedRegUnit(VRegOrUnit.asMCRegUnit());
-}
-
void RegPressureTracker::reset() {
MBB = nullptr;
LIS = nullptr;
@@ -577,21 +570,13 @@ void RegisterOperands::collect(const MachineInstr &MI,
}
void RegisterOperands::detectDeadDefs(const MachineInstr &MI,
- const LiveIntervals &LIS) {
- SlotIndex SlotIdx = LIS.getInstructionIndex(MI);
- for (auto *RI = Defs.begin(); RI != Defs.end(); /*empty*/) {
- const LiveRange *LR = getLiveRange(LIS, RI->VRegOrUnit);
- if (LR != nullptr) {
- LiveQueryResult LRQ = LR->Query(SlotIdx);
- if (LRQ.isDeadDef()) {
- // LiveIntervals knows this is a dead even though it's MachineOperand is
- // not flagged as such.
- DeadDefs.push_back(*RI);
- RI = Defs.erase(RI);
- continue;
- }
- }
- ++RI;
+ const LiveIntervals &LIS,
+ const MachineRegisterInfo &MRI) {
+ SlotIndex DeadSlotIdx = LIS.getInstructionIndex(MI).getDeadSlot();
+ for (auto *I = Defs.begin(); I != Defs.end(); /*empty*/) {
+ LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, /*TrackLaneMasks=*/false,
+ I->VRegOrUnit, DeadSlotIdx);
+ I = adjustDef(*I, LiveAfter);
}
}
@@ -623,22 +608,34 @@ void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS,
adjustUses(LIS, MRI, Pos);
+ const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
for (const VRegMaskOrUnit &P : DeadDefs) {
VirtRegOrUnit VRegOrUnit = P.VRegOrUnit;
if (!VRegOrUnit.isVirtualReg())
continue;
+ Register VReg = VRegOrUnit.asVirtualReg();
LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, /*TrackLaneMasks=*/true,
VRegOrUnit, Pos.getDeadSlot());
- if (LiveAfter.none())
- MI.setRegisterDefReadUndef(VRegOrUnit.asVirtualReg());
+ if (!LiveAfter.none())
+ continue;
+ // The register's read value doesn't matter if none of its lanes are live
+ // after the def.
+ MI.setRegisterDefReadUndef(VReg);
+
+ // The register's last definition should be marked dead.
+ const LiveInterval &LI = LIS.getInterval(VReg);
+ if (LI.segments.back().end == Pos.getDeadSlot())
+ MI.addRegisterDead(VReg, TRI, /*AddIfNotFound=*/false);
}
}
VRegMaskOrUnit *RegisterOperands::adjustDef(VRegMaskOrUnit &Def,
LaneBitmask LiveAfterDef) {
LaneBitmask ActualDef = Def.LaneMask & LiveAfterDef;
- if (ActualDef.none())
+ if (ActualDef.none()) {
+ DeadDefs.push_back(Def);
return Defs.erase(&Def);
+ }
Def.LaneMask = ActualDef;
return &Def + 1;
@@ -893,7 +890,7 @@ void RegPressureTracker::recede(SmallVectorImpl<VRegMaskOrUnit> *LiveUses) {
SlotIndex SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
} else if (RequireIntervals) {
- RegOpers.detectDeadDefs(MI, *LIS);
+ RegOpers.detectDeadDefs(MI, *LIS, *MRI);
}
recede(RegOpers, LiveUses);
@@ -1060,7 +1057,7 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
if (TrackLaneMasks)
RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
else if (RequireIntervals)
- RegOpers.detectDeadDefs(*MI, *LIS);
+ RegOpers.detectDeadDefs(*MI, *LIS, *MRI);
// Boost max pressure for all dead defs together.
// Since CurrSetPressure and MaxSetPressure
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 86ea8b0a98041..aa85be758559a 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2340,7 +2340,7 @@ void GCNSchedStage::modifyRegionSchedule(unsigned RegionIdx,
RegOpers.adjustLaneLiveness(*DAG.LIS, DAG.MRI, *MI);
} else {
// Adjust for missing dead-def flags.
- RegOpers.detectDeadDefs(*MI, *DAG.LIS);
+ RegOpers.detectDeadDefs(*MI, *DAG.LIS, DAG.MRI);
}
LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
}
diff --git a/llvm/unittests/CodeGen/CMakeLists.txt b/llvm/unittests/CodeGen/CMakeLists.txt
index deaa3dbcfcc5b..c192672e1a093 100644
--- a/llvm/unittests/CodeGen/CMakeLists.txt
+++ b/llvm/unittests/CodeGen/CMakeLists.txt
@@ -42,6 +42,7 @@ add_llvm_unittest(CodeGenTests
MIR2VecTest.cpp
RegAllocBasicTest.cpp
RegAllocScoreTest.cpp
+ RegisterPressureTest.cpp
RegisterTest.cpp
PassManagerTest.cpp
RematerializerTest.cpp
diff --git a/llvm/unittests/CodeGen/RegisterPressureTest.cpp b/llvm/unittests/CodeGen/RegisterPressureTest.cpp
new file mode 100644
index 0000000000000..e08a4c09931a3
--- /dev/null
+++ b/llvm/unittests/CodeGen/RegisterPressureTest.cpp
@@ -0,0 +1,91 @@
+//===- RegisterPressureTest.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/RegisterPressure.h"
+#include "CodeGenTestBase.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MIRParser/MIRParser.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/Config/Targets.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+class RegisterPressureTest : public CodeGenTestBase {
+public:
+ static void SetUpTestCase() {
+#if LLVM_HAS_AMDGPU_TARGET
+ LLVMInitializeAMDGPUTargetInfo();
+ LLVMInitializeAMDGPUTarget();
+ LLVMInitializeAMDGPUTargetMC();
+#else
+ GTEST_SKIP();
+#endif
+ }
+
+ void SetUp() override { setUpImpl("amdgpu9.50--", "", ""); }
+};
+
+/// Replicates the scheduler's effect on \p LIS on an intra-block move of \p
+/// MI right before \p MoveBefore, which must be in the same block as \p MI.
+static void moveMIAndAdjustLiveness(MachineBasicBlock::iterator MoveBefore,
+ MachineInstr &MI, LiveIntervals &LIS) {
+ MachineBasicBlock &MBB = *MI.getParent();
+ const MachineFunction &MF = *MBB.getParent();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+
+ MBB.splice(MoveBefore, &MBB, MI.getIterator());
+ LIS.handleMove(MI);
+
+ RegisterOperands RegOpers;
+ RegOpers.collect(MI, TRI, MRI, true, /*IgnoreDead=*/false);
+ RegOpers.adjustLaneLiveness(LIS, MRI, MI);
+}
+
+/// When a scheduler move turns a live subregister def into the dead, last def
+/// of its virtual register, `adjustLaneLiveness` must add a dead flag on the
+/// def operand. Here %0.sub1 defines a lane that is never used: before the move
+/// it is a partially-dead def sitting in the middle of %0's merged live
+/// interval (legal without a dead flag), but sinking it below the use of
+/// %0.sub0 makes it the interval's terminal segment, which ends on a dead slot.
+TEST_F(RegisterPressureTest, MarkDeadDefAfterMoveBeyondLastUse) {
+ StringRef MIRString = R"(
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ undef %0.sub0:vreg_64 = IMPLICIT_DEF
+ %0.sub1:vreg_64 = IMPLICIT_DEF
+ %1:vgpr_32 = V_ADD_U32_e32 %0.sub0, %0.sub0, implicit $exec
+
+ bb.1:
+ S_NOP 0, implicit %1
+ S_ENDPGM 0
+...
+ )";
+ ASSERT_TRUE(parseMIR(MIRString));
+
+ MachineFunction &MF = getMF("func");
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+
+ MachineBasicBlock &MBB0 = *MF.getBlockNumbered(0);
+ MachineInstr &Sub1Def = *std::next(MBB0.begin());
+
+ // Sink %0.sub1's (dead) def to the end of bb.0, past the use of %0.sub0. This
+ // replicates a scheduler move: the def becomes the last, dead def of %0.
+ moveMIAndAdjustLiveness(MBB0.end(), Sub1Def, LIS);
+
+ EXPECT_TRUE(MF.verify(&LIS, /*Indexes=*/nullptr, /*Banner=*/nullptr,
+ /*OS=*/&errs(), /*AbortOnError=*/false));
+}
More information about the llvm-commits
mailing list