[llvm] 355dac7 - [AMDGPU] Preserve liveness flags in ILP scheduler (#211190)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 00:56:04 PDT 2026
Author: Lukas Sommer
Date: 2026-07-23T09:56:00+02:00
New Revision: 355dac7c4997c891d0a917935dcccc80334f5392
URL: https://github.com/llvm/llvm-project/commit/355dac7c4997c891d0a917935dcccc80334f5392
DIFF: https://github.com/llvm/llvm-project/commit/355dac7c4997c891d0a917935dcccc80334f5392.diff
LOG: [AMDGPU] Preserve liveness flags in ILP scheduler (#211190)
During DAG construction, liveness flags such as `undef` are removed from
the instructions. When the scheduler succeeds, the flags are recomputed
and restored.
However, so far, if the scheduler failed to meet the occupancy target,
it would not restore the liveness flags, leading to verification
failure/assertions later in the pipeline.
This change restores the liveness flags in cases where no schedule
meeting the occupancy target could be found.
---------
Signed-off-by: Lukas Sommer <lukas.sommer at amd.com>
Added:
llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir
Modified:
llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
index dff153cebdd4c..a40896538b997 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
@@ -367,6 +367,26 @@ void GCNIterativeScheduler::scheduleBest(Region &R) {
R.BestSchedule.reset();
}
+void GCNIterativeScheduler::restoreLivenessFlags(MachineInstr &MI) {
+ assert(!MI.isDebugInstr());
+
+ for (MachineOperand &Op : MI.all_defs())
+ Op.setIsUndef(false);
+
+ RegisterOperands RegOpers;
+ RegOpers.collect(MI, *TRI, MRI, /*ShouldTrackLaneMasks=*/true,
+ /*IgnoreDead=*/false);
+ SlotIndex SlotIdx = LIS->getInstructionIndex(MI).getRegSlot();
+ RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, &MI);
+}
+
+void GCNIterativeScheduler::restoreRegionLivenessFlags(const Region &R) {
+ for (MachineBasicBlock::iterator I = R.Begin; I != R.End; ++I) {
+ if (!I->isDebugInstr())
+ restoreLivenessFlags(*I);
+ }
+}
+
// minimal required region scheduler, works for ranges of SUnits*,
// SUnits or MachineIntrs*
template <typename Range>
@@ -391,18 +411,8 @@ void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
if (NonDebugReordered)
LIS->handleMove(*MI, true);
}
- if (!MI->isDebugInstr()) {
- // Reset read - undef flags and update them later.
- for (auto &Op : MI->all_defs())
- Op.setIsUndef(false);
-
- RegisterOperands RegOpers;
- RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
- /*IgnoreDead*/false);
- // Adjust liveness and add missing dead+read-undef flags.
- auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
- RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
- }
+ if (!MI->isDebugInstr())
+ restoreLivenessFlags(*MI);
Top = std::next(MI->getIterator());
}
RegionBegin = getMachineInstr(Schedule.front());
@@ -622,6 +632,8 @@ void GCNIterativeScheduler::scheduleILP(
ST, DynamicVGPRBlockSize) >= TgtOcc) {
LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
scheduleBest(*R);
+ } else {
+ restoreRegionLivenessFlags(*R);
}
} else {
scheduleRegion(*R, ILPSchedule, RP);
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
index f731b1fc7e0df..448f31986a985 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
@@ -99,6 +99,11 @@ class GCNIterativeScheduler : public ScheduleDAGMILive {
const GCNRegPressure &MaxRP = GCNRegPressure());
void scheduleBest(Region &R);
+ // Restore the instruction's dead and read-undef liveness flags.
+ void restoreLivenessFlags(MachineInstr &MI);
+ // Restore dead and read-undef liveness flags for all instructions in the
+ // region.
+ void restoreRegionLivenessFlags(const Region &R);
std::vector<MachineInstr*> detachSchedule(ScheduleRef Schedule) const;
diff --git a/llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir b/llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir
new file mode 100644
index 0000000000000..2156861c640fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir
@@ -0,0 +1,49 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -run-pass=machine-scheduler -o - %s | FileCheck %s
+
+# Ensure that the scheduler preserves operand liveness and instruction flags
+# even if it fails to find a schedule that meets the occupancy requirement.
+--- |
+ target triple = "amdgpu9.42-amd-amdhsa"
+
+ define amdgpu_kernel void @test() #0 {
+ ret void
+ }
+
+ attributes #0 = { "amdgpu-sched-strategy"="iterative-ilp" }
+...
+---
+name: test
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 4
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test
+ ; CHECK: dead undef [[DEF:%[0-9]+]].sub0:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF1]]
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY1:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF2]]
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY2:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF3]]
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY3:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF4]]
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY4:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF5]]
+ ; CHECK-NEXT: dead [[S_ADD_U32_:%[0-9]+]]:sreg_32 = nuw S_ADD_U32 0, 1, implicit-def dead $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ undef %0.sub0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_1024_align2 = IMPLICIT_DEF
+ dead %2:vreg_1024_align2 = COPY %1
+ %3:vreg_1024_align2 = IMPLICIT_DEF
+ dead %4:vreg_1024_align2 = COPY %3
+ %5:vreg_1024_align2 = IMPLICIT_DEF
+ dead %6:vreg_1024_align2 = COPY %5
+ %7:vreg_1024_align2 = IMPLICIT_DEF
+ dead %8:vreg_1024_align2 = COPY %7
+ %9:vreg_1024_align2 = IMPLICIT_DEF
+ dead %10:vreg_1024_align2 = COPY %9
+ %11:sreg_32 = nuw S_ADD_U32 0, 1, implicit-def dead $scc
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list