https://github.com/fmayer created https://github.com/llvm/llvm-project/pull/238601
Assisted-By: Gemini
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 91a2816f1a49f6a8eb54c09ef5e725a1a7ae3fc8 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer at google.com>
Date: Tue, 3 Nov 2026 22:39:32 +0000
Subject: [PATCH] [MachinePipeliner] Fix PHIs with loop-invariant backedge
Assisted-By: Gemini
---
llvm/lib/CodeGen/ModuloSchedule.cpp | 18 ++++++++++++------
.../swp-phi-loop-invariant-backedge.mir | 9 +++++----
2 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index e0a33c73f052c..0ac825b819d6d 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -1406,13 +1406,14 @@ Register KernelRewriter::remapUse(Register Reg, MachineInstr &MI) {
if (!Producer)
return Reg;
+ if (Producer->getParent() != BB)
+ // Producer was not inside the loop. Use the register as-is.
+ return Reg;
+
int ConsumerStage = S.getStage(&MI);
if (!Producer->isPHI()) {
// Non-phi producers are simple to remap. Insert as many phis as the
// difference between the consumer and producer stages.
- if (Producer->getParent() != BB)
- // Producer was not inside the loop. Use the register as-is.
- return Reg;
int ProducerStage = S.getStage(Producer);
assert(ConsumerStage != -1 &&
"In-loop consumer should always be scheduled!");
@@ -1436,12 +1437,17 @@ Register KernelRewriter::remapUse(Register Reg, MachineInstr &MI) {
assert(LoopProducer);
}
int LoopProducerStage = S.getStage(LoopProducer);
+ if (LoopProducerStage == -1) {
+ assert(LoopProducer->getParent() != BB &&
+ "In-loop producer should always be scheduled!");
+ // The loop-carried value of the in-loop phi chain is defined outside the
+ // loop, so it is available at stage 0 of every iteration.
+ LoopProducerStage = 0;
+ }
std::optional<Register> IllegalPhiDefault;
- if (LoopProducerStage == -1) {
- // Do nothing.
- } else if (LoopProducerStage > ConsumerStage) {
+ if (LoopProducerStage > ConsumerStage) {
// This schedule is only representable if ProducerStage == ConsumerStage+1.
// In addition, Consumer's cycle must be scheduled after Producer in the
// rescheduled loop. This is enforced by the pipeliner's ASAP and ALAP
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-loop-invariant-backedge.mir b/llvm/test/CodeGen/Hexagon/swp-phi-loop-invariant-backedge.mir
index 6a18be80dba37..1799cfa1bc159 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-loop-invariant-backedge.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-loop-invariant-backedge.mir
@@ -29,7 +29,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[PHI:%[0-9]+]]:intregs = PHI [[L2_loadri_io]], %bb.3, %3, %bb.1
; CHECK-NEXT: [[PHI1:%[0-9]+]]:intregs = PHI [[A2_tfrsi1]], %bb.3, [[A2_tfrsi1]], %bb.1
- ; CHECK-NEXT: [[A2_add:%[0-9]+]]:intregs = A2_add [[PHI]], [[PHI1]]
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:intregs = PHI [[A2_tfrsi]], %bb.3, [[PHI1]], %bb.1
+ ; CHECK-NEXT: [[A2_add:%[0-9]+]]:intregs = A2_add [[PHI]], [[PHI2]]
; CHECK-NEXT: [[L2_loadri_io1:%[0-9]+]]:intregs = L2_loadri_io [[A2_tfrsi]], 0
; CHECK-NEXT: ENDLOOP0 %bb.1, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
; CHECK-NEXT: J2_jump %bb.5, implicit-def $pc
@@ -37,9 +38,9 @@ body: |
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.4(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:intregs = PHI [[L2_loadri_io1]], %bb.1
- ; CHECK-NEXT: [[PHI3:%[0-9]+]]:intregs = PHI [[A2_tfrsi1]], %bb.1
- ; CHECK-NEXT: [[A2_add1:%[0-9]+]]:intregs = A2_add [[PHI2]], [[PHI3]]
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:intregs = PHI [[L2_loadri_io1]], %bb.1
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:intregs = PHI [[PHI1]], %bb.1
+ ; CHECK-NEXT: [[A2_add1:%[0-9]+]]:intregs = A2_add [[PHI3]], [[PHI4]]
; CHECK-NEXT: J2_jump %bb.4, implicit-def $pc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4: