[llvm] [llvm][CodeGen] avoid repeated interval calculation in window scheduler (PR #132352)
Hua Tian via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 1 23:32:02 PDT 2025
https://github.com/huaatian updated https://github.com/llvm/llvm-project/pull/132352
>From 993262e95c50cf16e1c02ac1517c83d7647ea09e Mon Sep 17 00:00:00 2001
From: akiratian <akiratian at tencent.com>
Date: Fri, 21 Mar 2025 15:21:01 +0800
Subject: [PATCH 1/3] [llvm][CodeGen] avoid repeated interval calculation in
window scheduler
Some new registers may be reused when replacing some old ones in
certain use case of ModuloScheduleExpander. It is necessary to
avoid repeated interval calculations for these registers.
---
llvm/include/llvm/CodeGen/ModuloSchedule.h | 3 +-
llvm/lib/CodeGen/ModuloSchedule.cpp | 10 +-
.../AArch64/aarch64-swp-ws-live-intervals.mir | 103 ++++++++++++++++++
3 files changed, 110 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index b6000ba05d882..a1b949844371f 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -60,6 +60,7 @@
#ifndef LLVM_CODEGEN_MODULOSCHEDULE_H
#define LLVM_CODEGEN_MODULOSCHEDULE_H
+#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopUtils.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
@@ -189,7 +190,7 @@ class ModuloScheduleExpander {
InstrChangesTy InstrChanges;
/// Record the registers that need to compute live intervals.
- SmallVector<Register> NoIntervalRegs;
+ SmallSet<Register, 8> NoIntervalRegs;
void generatePipelinedLoop();
void generateProlog(unsigned LastStage, MachineBasicBlock *KernelBB,
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index d208a62a99372..50fc0f437a5c5 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -551,7 +551,7 @@ void ModuloScheduleExpander::generateExistingPhis(
if (IsLast && np == NumPhis - 1) {
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
+ NoIntervalRegs.insert(NewReg);
}
continue;
}
@@ -594,7 +594,7 @@ void ModuloScheduleExpander::generateExistingPhis(
// epilog.
if (IsLast && np == NumPhis - 1) {
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
+ NoIntervalRegs.insert(NewReg);
}
// In the kernel, a dependent Phi uses the value from this Phi.
@@ -617,7 +617,7 @@ void ModuloScheduleExpander::generateExistingPhis(
auto It = CurStageMap.find(LoopVal);
if (It != CurStageMap.end()) {
replaceRegUsesAfterLoop(Def, It->second, BB, MRI);
- NoIntervalRegs.push_back(It->second);
+ NoIntervalRegs.insert(It->second);
}
}
}
@@ -740,7 +740,7 @@ void ModuloScheduleExpander::generatePhis(
}
if (IsLast && np == NumPhis - 1) {
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
+ NoIntervalRegs.insert(NewReg);
}
}
}
@@ -1083,7 +1083,7 @@ void ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
VRMap[CurStageNum][reg] = NewReg;
if (LastDef) {
replaceRegUsesAfterLoop(reg, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
+ NoIntervalRegs.insert(NewReg);
}
} else if (MO.isUse()) {
MachineInstr *Def = MRI.getVRegDef(reg);
diff --git a/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir b/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
new file mode 100644
index 0000000000000..4f0e52239b90c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
@@ -0,0 +1,103 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc --mtriple=aarch64 %s -O2 -run-pass=pipeliner -o - | FileCheck %s
+
+...
+---
+name: foo
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: foo
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[FMOVD0_:%[0-9]+]]:fpr64 = FMOVD0
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64sp = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[FADDDrr:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FMOVD0_]], [[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[SUBSXri:%[0-9]+]]:gpr64 = nsw SUBSXri [[SUBREG_TO_REG]], 1, 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[SUBSXri]]
+ ; CHECK-NEXT: [[FMOVDi:%[0-9]+]]:fpr64 = FMOVDi 112
+ ; CHECK-NEXT: Bcc 0, %bb.7, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000), %bb.6(0x00000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[FADDDrr1:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FADDDrr]], [[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDDrr2:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FMOVD0_]], [[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[SUBSXri1:%[0-9]+]]:gpr64 = nsw SUBSXri [[COPY1]], 1, 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64all = COPY [[SUBSXri1]]
+ ; CHECK-NEXT: [[FMOVDi1:%[0-9]+]]:fpr64 = FMOVDi 112
+ ; CHECK-NEXT: Bcc 0, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x04000000), %bb.5(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64sp = PHI [[COPY2]], %bb.4, %24, %bb.5
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:fpr64 = PHI [[FMOVDi1]], %bb.4, %25, %bb.5
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:fpr64 = PHI [[FMOVDi]], %bb.4, [[PHI1]], %bb.5
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:fpr64 = PHI [[FADDDrr2]], %bb.4, %22, %bb.5
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:fpr64 = PHI [[FADDDrr1]], %bb.4, %23, %bb.5
+ ; CHECK-NEXT: [[SUBSXri2:%[0-9]+]]:gpr64 = nsw SUBSXri [[PHI]], 1, 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[FADDDrr3:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI2]], [[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDDrr4:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI3]], [[PHI2]], implicit $fpcr
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64all = COPY [[SUBSXri2]]
+ ; CHECK-NEXT: STRDui [[PHI4]], [[COPY]], 0
+ ; CHECK-NEXT: [[FMOVDi2:%[0-9]+]]:fpr64 = FMOVDi 112
+ ; CHECK-NEXT: Bcc 1, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI5:%[0-9]+]]:fpr64 = PHI [[FMOVDi]], %bb.4, [[PHI1]], %bb.5
+ ; CHECK-NEXT: [[PHI6:%[0-9]+]]:fpr64 = PHI [[FADDDrr2]], %bb.4, [[FADDDrr3]], %bb.5
+ ; CHECK-NEXT: [[PHI7:%[0-9]+]]:fpr64 = PHI [[FADDDrr1]], %bb.4, [[FADDDrr4]], %bb.5
+ ; CHECK-NEXT: STRDui [[PHI7]], [[COPY]], 0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI8:%[0-9]+]]:fpr64 = PHI [[FMOVD0_]], %bb.3, [[PHI5]], %bb.6
+ ; CHECK-NEXT: [[PHI9:%[0-9]+]]:fpr64 = PHI [[FADDDrr]], %bb.3, [[PHI6]], %bb.6
+ ; CHECK-NEXT: [[FADDDrr5:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI9]], [[PHI8]], implicit $fpcr
+ ; CHECK-NEXT: STRDui [[FADDDrr5]], [[COPY]], 0
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+ %1:fpr64 = FMOVD0
+ %2:gpr32 = MOVi32imm 1
+ %3:gpr64all = SUBREG_TO_REG 0, killed %2, %subreg.sub_32
+
+ bb.1:
+ successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+ %4:gpr64sp = PHI %3, %bb.0, %5, %bb.1
+ %6:fpr64 = PHI %1, %bb.0, %7, %bb.1
+ %8:fpr64 = PHI %1, %bb.0, %6, %bb.1
+ %9:fpr64 = nofpexcept FADDDrr %8, %1, implicit $fpcr
+ %10:fpr64 = nofpexcept FADDDrr killed %9, %6, implicit $fpcr
+ STRDui killed %10, %0, 0
+ %11:gpr64 = nsw SUBSXri %4, 1, 0, implicit-def $nzcv
+ %5:gpr64all = COPY %11
+ %7:fpr64 = FMOVDi 112
+ Bcc 1, %bb.1, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ RET_ReallyLR
+
+...
>From 7efdd4ff71449c3027a05b5478eb87651aba42ab Mon Sep 17 00:00:00 2001
From: akiratian <akiratian at tencent.com>
Date: Fri, 28 Mar 2025 15:16:22 +0800
Subject: [PATCH 2/3] [llvm][CodeGen] Modifications made based on review
comments 1
---
llvm/include/llvm/CodeGen/ModuloSchedule.h | 4 ---
llvm/lib/CodeGen/ModuloSchedule.cpp | 32 ++++------------------
2 files changed, 5 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index a1b949844371f..2dc6b2d88cedc 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -189,9 +189,6 @@ class ModuloScheduleExpander {
/// Instructions to change when emitting the final schedule.
InstrChangesTy InstrChanges;
- /// Record the registers that need to compute live intervals.
- SmallSet<Register, 8> NoIntervalRegs;
-
void generatePipelinedLoop();
void generateProlog(unsigned LastStage, MachineBasicBlock *KernelBB,
ValueMapTy *VRMap, MBBVectorTy &PrologBBs);
@@ -215,7 +212,6 @@ class ModuloScheduleExpander {
void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs,
MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
ValueMapTy *VRMap);
- void calculateIntervals();
bool computeDelta(MachineInstr &MI, unsigned &Delta);
void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI,
unsigned Num);
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index 50fc0f437a5c5..352093ab6bdf9 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -181,10 +181,6 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
// Add branches between prolog and epilog blocks.
addBranches(*Preheader, PrologBBs, KernelBB, EpilogBBs, VRMap);
- // The intervals of newly created virtual registers are calculated after the
- // kernel expansion.
- calculateIntervals();
-
delete[] VRMap;
delete[] VRMapPhi;
}
@@ -549,10 +545,8 @@ void ModuloScheduleExpander::generateExistingPhis(
if (VRMap[LastStageNum - np - 1].count(LoopVal))
PhiOp2 = VRMap[LastStageNum - np - 1][LoopVal];
- if (IsLast && np == NumPhis - 1) {
+ if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.insert(NewReg);
- }
continue;
}
}
@@ -592,10 +586,8 @@ void ModuloScheduleExpander::generateExistingPhis(
// Check if we need to rename any uses that occurs after the loop. The
// register to replace depends on whether the Phi is scheduled in the
// epilog.
- if (IsLast && np == NumPhis - 1) {
+ if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.insert(NewReg);
- }
// In the kernel, a dependent Phi uses the value from this Phi.
if (InKernel)
@@ -615,10 +607,8 @@ void ModuloScheduleExpander::generateExistingPhis(
if (NumStages == 0 && IsLast) {
auto &CurStageMap = VRMap[CurStageNum];
auto It = CurStageMap.find(LoopVal);
- if (It != CurStageMap.end()) {
+ if (It != CurStageMap.end())
replaceRegUsesAfterLoop(Def, It->second, BB, MRI);
- NoIntervalRegs.insert(It->second);
- }
}
}
}
@@ -738,10 +728,8 @@ void ModuloScheduleExpander::generatePhis(
rewriteScheduledInstr(NewBB, InstrMap, CurStageNum, np, &*BBI, Def,
NewReg);
}
- if (IsLast && np == NumPhis - 1) {
+ if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.insert(NewReg);
- }
}
}
}
@@ -953,14 +941,6 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
}
}
-/// Some registers are generated during the kernel expansion. We calculate the
-/// live intervals of these registers after the expansion.
-void ModuloScheduleExpander::calculateIntervals() {
- for (Register Reg : NoIntervalRegs)
- LIS.createAndComputeVirtRegInterval(Reg);
- NoIntervalRegs.clear();
-}
-
/// Return true if we can compute the amount the instruction changes
/// during each iteration. Set Delta to the amount of the change.
bool ModuloScheduleExpander::computeDelta(MachineInstr &MI, unsigned &Delta) {
@@ -1081,10 +1061,8 @@ void ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
Register NewReg = MRI.createVirtualRegister(RC);
MO.setReg(NewReg);
VRMap[CurStageNum][reg] = NewReg;
- if (LastDef) {
+ if (LastDef)
replaceRegUsesAfterLoop(reg, NewReg, BB, MRI);
- NoIntervalRegs.insert(NewReg);
- }
} else if (MO.isUse()) {
MachineInstr *Def = MRI.getVRegDef(reg);
// Compute the stage that contains the last definition for instruction.
>From 394f1db63e97f00ef4e11d8a643e343843ce505a Mon Sep 17 00:00:00 2001
From: akiratian <akiratian at tencent.com>
Date: Wed, 2 Apr 2025 14:31:26 +0800
Subject: [PATCH 3/3] [llvm][CodeGen] Modifications made based on review
comments 2
---
llvm/include/llvm/CodeGen/ModuloSchedule.h | 1 -
llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index 2dc6b2d88cedc..49dc746d3ee35 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -60,7 +60,6 @@
#ifndef LLVM_CODEGEN_MODULOSCHEDULE_H
#define LLVM_CODEGEN_MODULOSCHEDULE_H
-#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopUtils.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
diff --git a/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir b/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
index 4f0e52239b90c..48f02452e3597 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
+++ b/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc --mtriple=aarch64 %s -O2 -run-pass=pipeliner -o - | FileCheck %s
+# RUN: llc --mtriple=aarch64 %s -run-pass=pipeliner -o - | FileCheck %s
...
---
More information about the llvm-commits
mailing list