[llvm] [AMDGPU] Preserve instruction flags in ILP scheduler (PR #211190)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 00:48:01 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Lukas Sommer (sommerlukas)
<details>
<summary>Changes</summary>
During DAG construction, flags such as `undef` are removed from the instructions. When the scheduler succeeds, the flags are recomputed and restored.
However, so far, if the scheduler failed to meet the occupancy target, it would not restore the flags, leading to verification failure/assertions later in the pipeline.
This change restores the flags in cases where no schedule meeting the occupancy target could be found.
---
Full diff: https://github.com/llvm/llvm-project/pull/211190.diff
3 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp (+23-13)
- (modified) llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h (+5)
- (added) llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir (+47)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
index dff153cebdd4c..8fcca67529995 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
@@ -367,6 +367,25 @@ void GCNIterativeScheduler::scheduleBest(Region &R) {
R.BestSchedule.reset();
}
+void GCNIterativeScheduler::restoreFlags(MachineInstr &MI) {
+ assert(!MI.isDebugInstr());
+
+ for (MachineOperand &Op : MI.all_defs())
+ Op.setIsUndef(false);
+
+ RegisterOperands RegOpers;
+ RegOpers.collect(MI, *TRI, MRI, /*ShouldTrackLaneMasks=*/true,
+ /*IgnoreDead=*/false);
+ SlotIndex SlotIdx = LIS->getInstructionIndex(MI).getRegSlot();
+ RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, &MI);
+}
+
+void GCNIterativeScheduler::restoreRegionFlags(const Region &R) {
+ for (auto I = R.Begin; I != R.End; ++I)
+ if (!I->isDebugInstr())
+ restoreFlags(*I);
+}
+
// minimal required region scheduler, works for ranges of SUnits*,
// SUnits or MachineIntrs*
template <typename Range>
@@ -391,18 +410,8 @@ void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
if (NonDebugReordered)
LIS->handleMove(*MI, true);
}
- if (!MI->isDebugInstr()) {
- // Reset read - undef flags and update them later.
- for (auto &Op : MI->all_defs())
- Op.setIsUndef(false);
-
- RegisterOperands RegOpers;
- RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
- /*IgnoreDead*/false);
- // Adjust liveness and add missing dead+read-undef flags.
- auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
- RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
- }
+ if (!MI->isDebugInstr())
+ restoreFlags(*MI);
Top = std::next(MI->getIterator());
}
RegionBegin = getMachineInstr(Schedule.front());
@@ -622,7 +631,8 @@ void GCNIterativeScheduler::scheduleILP(
ST, DynamicVGPRBlockSize) >= TgtOcc) {
LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
scheduleBest(*R);
- }
+ } else
+ restoreRegionFlags(*R);
} else {
scheduleRegion(*R, ILPSchedule, RP);
LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
index f731b1fc7e0df..f88d7c53d1da4 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.h
@@ -99,6 +99,11 @@ class GCNIterativeScheduler : public ScheduleDAGMILive {
const GCNRegPressure &MaxRP = GCNRegPressure());
void scheduleBest(Region &R);
+ // Adjust liveness and add missing dead+read-undef flags for the instruction.
+ void restoreFlags(MachineInstr &MI);
+ // Adjust liveness and add missing dead+read-undef flags for all the
+ // instructions in the region.
+ void restoreRegionFlags(const Region &R);
std::vector<MachineInstr*> detachSchedule(ScheduleRef Schedule) const;
diff --git a/llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir b/llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir
new file mode 100644
index 0000000000000..aeeab236d08b0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/iterative-ilp-restore-undef.mir
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=machine-scheduler -o - %s | FileCheck %s
+
+# Ensure that the scheduler preserves flags such as `undef` even if it fails to
+# find a schedule that meets the occupancy requirement.
+--- |
+ target triple = "amdgcn-amd-amdhsa"
+
+ define amdgpu_kernel void @test() #0 {
+ ret void
+ }
+
+ attributes #0 = { "amdgpu-sched-strategy"="iterative-ilp" "target-cpu"="gfx942" }
+...
+---
+name: test
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 4
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test
+ ; CHECK: dead undef [[DEF:%[0-9]+]].sub0:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF1]]
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY1:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF2]]
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY2:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF3]]
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY3:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF4]]
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vreg_1024_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead [[COPY4:%[0-9]+]]:vreg_1024_align2 = COPY [[DEF5]]
+ ; CHECK-NEXT: S_ENDPGM 0
+ undef %0.sub0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_1024_align2 = IMPLICIT_DEF
+ dead %2:vreg_1024_align2 = COPY %1
+ %3:vreg_1024_align2 = IMPLICIT_DEF
+ dead %4:vreg_1024_align2 = COPY %3
+ %5:vreg_1024_align2 = IMPLICIT_DEF
+ dead %6:vreg_1024_align2 = COPY %5
+ %7:vreg_1024_align2 = IMPLICIT_DEF
+ dead %8:vreg_1024_align2 = COPY %7
+ %9:vreg_1024_align2 = IMPLICIT_DEF
+ dead %10:vreg_1024_align2 = COPY %9
+ S_ENDPGM 0
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/211190
More information about the llvm-commits
mailing list