[llvm] [CodeGen][MISched] Add misched post-regalloc bidirectional scheduling (PR #77138)
Michael Maitland via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 12 07:31:15 PDT 2024
https://github.com/michaelmaitland updated https://github.com/llvm/llvm-project/pull/77138
>From da1d9ae923b5f6ea412cd85051d610001443f5ae Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland at gmail.com>
Date: Fri, 5 Jan 2024 12:46:25 -0800
Subject: [PATCH 1/3] [CodeGen][MISched] Add misched post-regalloc
bidirectional scheduling
This PR is stacked on #76186.
This PR keeps the default strategy as top-down since that is what
existing targets expect. It can be enabled using
`-misched-postra-direction=bidirectional`.
It is up to targets to decide whether they would like to enable this
option for themselves.
---
llvm/include/llvm/CodeGen/MachineScheduler.h | 9 ++
llvm/lib/CodeGen/MachineScheduler.cpp | 117 +++++++++++++++---
.../RISCV/misched-postra-direction.mir | 17 ++-
3 files changed, 127 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 25703dd6b61f0d..9cca6b3a571c3b 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -1296,6 +1296,11 @@ class PostGenericScheduler : public GenericSchedulerBase {
SchedBoundary Bot;
MachineSchedPolicy RegionPolicy;
+ /// Candidate last picked from Top boundary.
+ SchedCandidate TopCand;
+ /// Candidate last picked from Bot boundary.
+ SchedCandidate BotCand;
+
public:
PostGenericScheduler(const MachineSchedContext *C)
: GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ"),
@@ -1316,6 +1321,8 @@ class PostGenericScheduler : public GenericSchedulerBase {
SUnit *pickNode(bool &IsTopNode) override;
+ SUnit *pickNodeBidirectional(bool &IsTopNode);
+
void scheduleTree(unsigned SubtreeID) override {
llvm_unreachable("PostRA scheduler does not support subtree analysis.");
}
@@ -1326,12 +1333,14 @@ class PostGenericScheduler : public GenericSchedulerBase {
if (SU->isScheduled)
return;
Top.releaseNode(SU, SU->TopReadyCycle, false);
+ TopCand.SU = nullptr;
}
void releaseBottomNode(SUnit *SU) override {
if (SU->isScheduled)
return;
Bot.releaseNode(SU, SU->BotReadyCycle, false);
+ BotCand.SU = nullptr;
}
protected:
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 3bbd126bdaf1af..a2fcfaec3da2f1 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -85,6 +85,7 @@ namespace MISchedPostRASched {
enum Direction {
TopDown,
BottomUp,
+ Bidirectional,
};
} // end namespace MISchedPostRASched
cl::opt<MISchedPostRASched::Direction> PostRADirection(
@@ -93,10 +94,13 @@ cl::opt<MISchedPostRASched::Direction> PostRADirection(
// Default to top-down because it was implemented first and existing targets
// expect that behavior by default.
cl::init(MISchedPostRASched::TopDown),
- cl::values(clEnumValN(MISchedPostRASched::TopDown, "topdown",
- "Force top-down post reg-alloc list scheduling"),
- clEnumValN(MISchedPostRASched::BottomUp, "bottomup",
- "Force bottom-up post reg-alloc list scheduling")));
+ cl::values(
+ clEnumValN(MISchedPostRASched::TopDown, "topdown",
+ "Force top-down post reg-alloc list scheduling"),
+ clEnumValN(MISchedPostRASched::BottomUp, "bottomup",
+ "Force bottom-up post reg-alloc list scheduling"),
+ clEnumValN(MISchedPostRASched::Bidirectional, "bidirectional",
+ "Force bidirectional post reg-alloc list scheduling")));
cl::opt<bool>
DumpCriticalPathLength("misched-dcpl", cl::Hidden,
cl::desc("Print critical path length to stdout"));
@@ -497,11 +501,13 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
// Instantiate the selected scheduler for this target, function, and
// optimization level.
std::unique_ptr<ScheduleDAGInstrs> Scheduler(createPostMachineScheduler());
- ScheduleDAGMI::DumpDirection D;
+ ScheduleDAGInstrs::DumpDirection D;
if (PostRADirection == MISchedPostRASched::TopDown)
D = ScheduleDAGMI::DumpDirection::TopDown;
- else
+ else if (PostRADirection == MISchedPostRASched::BottomUp)
D = ScheduleDAGMI::DumpDirection::BottomUp;
+ else
+ D = ScheduleDAGMI::DumpDirection::Bidirectional;
Scheduler->setDumpDirection(D);
scheduleRegions(*Scheduler, true);
@@ -3694,7 +3700,7 @@ SUnit *GenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
TCand.reset(CandPolicy());
pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand);
assert(TCand.SU == BotCand.SU &&
- "Last pick result should correspond to re-picking right now");
+ "Last pick result should correspond to re-picking right now");
}
#endif
}
@@ -3714,7 +3720,7 @@ SUnit *GenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
TCand.reset(CandPolicy());
pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand);
assert(TCand.SU == TopCand.SU &&
- "Last pick result should correspond to re-picking right now");
+ "Last pick result should correspond to re-picking right now");
}
#endif
}
@@ -3890,6 +3896,9 @@ void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin,
} else if (PostRADirection == MISchedPostRASched::BottomUp) {
RegionPolicy.OnlyTopDown = false;
RegionPolicy.OnlyBottomUp = true;
+ } else if (PostRADirection == MISchedPostRASched::Bidirectional) {
+ RegionPolicy.OnlyBottomUp = false;
+ RegionPolicy.OnlyTopDown = false;
}
}
@@ -3969,6 +3978,87 @@ void PostGenericScheduler::pickNodeFromQueue(SchedBoundary &Zone,
}
}
+/// Pick the best candidate node from either the top or bottom queue.
+SUnit *PostGenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
+ // FIXME: This is similiar to GenericScheduler::pickNodeBidirectional. Factor
+ // out common parts.
+
+ // Schedule as far as possible in the direction of no choice. This is most
+ // efficient, but also provides the best heuristics for CriticalPSets.
+ if (SUnit *SU = Bot.pickOnlyChoice()) {
+ IsTopNode = false;
+ tracePick(Only1, false);
+ return SU;
+ }
+ if (SUnit *SU = Top.pickOnlyChoice()) {
+ IsTopNode = true;
+ tracePick(Only1, true);
+ return SU;
+ }
+ // Set the bottom-up policy based on the state of the current bottom zone and
+ // the instructions outside the zone, including the top zone.
+ CandPolicy BotPolicy;
+ setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
+ // Set the top-down policy based on the state of the current top zone and
+ // the instructions outside the zone, including the bottom zone.
+ CandPolicy TopPolicy;
+ setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
+
+ // See if BotCand is still valid (because we previously scheduled from Top).
+ LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
+ if (!BotCand.isValid() || BotCand.SU->isScheduled ||
+ BotCand.Policy != BotPolicy) {
+ BotCand.reset(CandPolicy());
+ pickNodeFromQueue(Bot, BotCand);
+ assert(BotCand.Reason != NoCand && "failed to find the first candidate");
+ } else {
+ LLVM_DEBUG(traceCandidate(BotCand));
+#ifndef NDEBUG
+ if (VerifyScheduling) {
+ SchedCandidate TCand;
+ TCand.reset(CandPolicy());
+ pickNodeFromQueue(Bot, BotCand);
+ assert(TCand.SU == BotCand.SU &&
+ "Last pick result should correspond to re-picking right now");
+ }
+#endif
+ }
+
+ // Check if the top Q has a better candidate.
+ LLVM_DEBUG(dbgs() << "Picking from Top:\n");
+ if (!TopCand.isValid() || TopCand.SU->isScheduled ||
+ TopCand.Policy != TopPolicy) {
+ TopCand.reset(CandPolicy());
+ pickNodeFromQueue(Top, TopCand);
+ assert(TopCand.Reason != NoCand && "failed to find the first candidate");
+ } else {
+ LLVM_DEBUG(traceCandidate(TopCand));
+#ifndef NDEBUG
+ if (VerifyScheduling) {
+ SchedCandidate TCand;
+ TCand.reset(CandPolicy());
+ pickNodeFromQueue(Top, TopCand);
+ assert(TCand.SU == TopCand.SU &&
+ "Last pick result should correspond to re-picking right now");
+ }
+#endif
+ }
+
+ // Pick best from BotCand and TopCand.
+ assert(BotCand.isValid());
+ assert(TopCand.isValid());
+ SchedCandidate Cand = BotCand;
+ TopCand.Reason = NoCand;
+ if (tryCandidate(Cand, TopCand)) {
+ Cand.setBest(TopCand);
+ LLVM_DEBUG(traceCandidate(Cand));
+ }
+
+ IsTopNode = Cand.AtTop;
+ tracePick(Cand);
+ return Cand.SU;
+}
+
/// Pick the next node to schedule.
SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
if (DAG->top() == DAG->bottom()) {
@@ -3979,13 +4069,12 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
SUnit *SU;
do {
if (RegionPolicy.OnlyBottomUp) {
- assert(!RegionPolicy.OnlyTopDown);
SU = Bot.pickOnlyChoice();
if (SU) {
tracePick(Only1, true);
} else {
CandPolicy NoPolicy;
- SchedCandidate BotCand(NoPolicy);
+ BotCand.reset(NoPolicy);
// Set the bottom-up policy based on the state of the current bottom
// zone and the instructions outside the zone, including the top zone.
setPolicy(BotCand.Policy, /*IsPostRA=*/true, Bot, nullptr);
@@ -3995,15 +4084,13 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
SU = BotCand.SU;
}
IsTopNode = false;
- } else {
-
- assert(RegionPolicy.OnlyTopDown);
+ } else if (RegionPolicy.OnlyTopDown) {
SU = Top.pickOnlyChoice();
if (SU) {
tracePick(Only1, true);
} else {
CandPolicy NoPolicy;
- SchedCandidate TopCand(NoPolicy);
+ TopCand.reset(NoPolicy);
// Set the top-down policy based on the state of the current top zone
// and the instructions outside the zone, including the bottom zone.
setPolicy(TopCand.Policy, /*IsPostRA=*/true, Top, nullptr);
@@ -4013,6 +4100,8 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
SU = TopCand.SU;
}
IsTopNode = true;
+ } else {
+ SU = pickNodeBidirectional(IsTopNode);
}
} while (SU->isScheduled);
diff --git a/llvm/test/CodeGen/RISCV/misched-postra-direction.mir b/llvm/test/CodeGen/RISCV/misched-postra-direction.mir
index 841d0e6d65da69..a4cf150d44d7b6 100644
--- a/llvm/test/CodeGen/RISCV/misched-postra-direction.mir
+++ b/llvm/test/CodeGen/RISCV/misched-postra-direction.mir
@@ -1,5 +1,15 @@
-# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=topdown -o - %s 2>&1 | FileCheck --check-prefix=TOPDOWN %s
-# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=bottomup -o - %s 2>&1 | FileCheck --check-prefix=BOTTOMUP %s
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN: -enable-post-misched -debug-only=machine-scheduler \
+# RUN: -misched-dump-schedule-trace -misched-postra-direction=topdown \
+# RUN: -o - %s 2>&1 | FileCheck --check-prefix=TOPDOWN %s
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN: -enable-post-misched -debug-only=machine-scheduler \
+# RUN: -misched-dump-schedule-trace -misched-postra-direction=bottomup \
+# RUN: -o - %s 2>&1 | FileCheck --check-prefix=BOTTOMUP %s
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN: -enable-post-misched -debug-only=machine-scheduler \
+# RUN: -misched-dump-schedule-trace -misched-postra-direction=bidirectional \
+# RUN: -o - %s 2>&1 | FileCheck --check-prefix=BIDIRECTIONAL %s
# REQUIRES: asserts
@@ -51,3 +61,6 @@ body: |
# BOTTOMUP-NEXT: SU(1): renamable $x13 = ADD renamable $x11, renamable $x10
# BOTTOMUP-NEXT: SU(0): renamable $x12 = MUL renamable $x11, renamable $x10
# BOTTOMUP-NEXT: SU(2): renamable $x14 = DIVW renamable $x12, renamable $x13
+
+# BIDIRECTIONAL: *** Final schedule for %bb.0 ***
+# BIDIRECTIONAL-NEXT: * Schedule table (Bidirectional): not implemented
>From fab89600ac91e54a772798b63a1d0baa922b2e22 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland at gmail.com>
Date: Tue, 27 Feb 2024 10:13:29 -0800
Subject: [PATCH 2/3] fixup! get clang format to stop complaining on the PR
---
llvm/lib/CodeGen/MachineScheduler.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index a2fcfaec3da2f1..4fc014ac8ee89b 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -3700,7 +3700,7 @@ SUnit *GenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
TCand.reset(CandPolicy());
pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand);
assert(TCand.SU == BotCand.SU &&
- "Last pick result should correspond to re-picking right now");
+ "Last pick result should correspond to re-picking right now");
}
#endif
}
@@ -4039,7 +4039,7 @@ SUnit *PostGenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
TCand.reset(CandPolicy());
pickNodeFromQueue(Top, TopCand);
assert(TCand.SU == TopCand.SU &&
- "Last pick result should correspond to re-picking right now");
+ "Last pick result should correspond to re-picking right now");
}
#endif
}
>From 70b791f8a7824754b95963caebede3dfa1d96471 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland at gmail.com>
Date: Tue, 12 Mar 2024 07:30:57 -0700
Subject: [PATCH 3/3] fixup! IsPostRA=true
---
llvm/lib/CodeGen/MachineScheduler.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 4fc014ac8ee89b..a7b7fff78024fe 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -3998,11 +3998,11 @@ SUnit *PostGenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
// Set the bottom-up policy based on the state of the current bottom zone and
// the instructions outside the zone, including the top zone.
CandPolicy BotPolicy;
- setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
+ setPolicy(BotPolicy, /*IsPostRA=*/true, Bot, &Top);
// Set the top-down policy based on the state of the current top zone and
// the instructions outside the zone, including the bottom zone.
CandPolicy TopPolicy;
- setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
+ setPolicy(TopPolicy, /*IsPostRA=*/true, Top, &Bot);
// See if BotCand is still valid (because we previously scheduled from Top).
LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
More information about the llvm-commits
mailing list