[llvm-branch-commits] [llvm] [MachinePipelner] Add loop-carried dependencies for global barriers (PR #174391)
Ryotaro Kasuga via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jan 5 03:30:45 PST 2026
https://github.com/kasuga-fj created https://github.com/llvm/llvm-project/pull/174391
None
>From 02c7724b8e401171ac789ee27e8b4cbbe7788161 Mon Sep 17 00:00:00 2001
From: Ryotaro Kasuga <kasuga.ryotaro at fujitsu.com>
Date: Fri, 26 Dec 2025 07:04:55 +0000
Subject: [PATCH] [MachinePipelner] Add loop-carried dependencies for global
barriers
---
llvm/lib/CodeGen/MachinePipeliner.cpp | 50 +++++++++++++++++--
.../sms-loop-carried-fp-exceptions1.mir | 12 ++++-
.../Hexagon/swp-loop-carried-order-dep6.mir | 40 +++++++++------
3 files changed, 81 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 6b022783f4bb8..e5376b06f83a2 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -349,6 +349,10 @@ class LoopCarriedOrderDepsTracker {
const SUnitWithMemInfo &Dst);
void computeDependenciesAux();
+
+ void setLoopCarriedDep(const SUnit *Src, const SUnit *Dst) {
+ LoopCarried[Src->NodeNum].set(Dst->NodeNum);
+ }
};
} // end anonymous namespace
@@ -1137,7 +1141,7 @@ void LoopCarriedOrderDepsTracker::addDependenciesBetweenSUs(
return;
if (hasLoopCarriedMemDep(Src, Dst, *BAA, TII, TRI, DAG))
- LoopCarried[Src.SU->NodeNum].set(Dst.SU->NodeNum);
+ setLoopCarriedDep(Src.SU, Dst.SU);
}
void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks(
@@ -1160,11 +1164,16 @@ void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks(
void LoopCarriedOrderDepsTracker::computeDependenciesAux() {
SmallVector<LoadStoreChunk, 2> Chunks(1);
+ SUnit *FirstBarrier = nullptr;
+ SUnit *LastBarrier = nullptr;
for (const auto &TSU : TaggedSUnits) {
InstrTag Tag = TSU.getTag();
SUnit *SU = TSU.getPointer();
switch (Tag) {
case InstrTag::Barrier:
+ if (!FirstBarrier)
+ FirstBarrier = SU;
+ LastBarrier = SU;
Chunks.emplace_back();
break;
case InstrTag::LoadOrStore:
@@ -1182,9 +1191,42 @@ void LoopCarriedOrderDepsTracker::computeDependenciesAux() {
for (const LoadStoreChunk &Chunk : Chunks)
addLoopCarriedDepenenciesForChunks(Chunk, Chunk);
- // TODO: If there are multiple barrier instructions, dependencies from the
- // last barrier instruction (or load/store below it) to the first barrier
- // instruction (or load/store above it).
+ // There is no barrier instruction between load/store instructions in the same
+ // LoadStoreChunk. If there are one or more barrier instructions, the
+ // instructions sequence is as follows:
+ //
+ // Loads/Stores (Chunks.front())
+ // Barrier (FirstBarrier)
+ // Loads/Stores
+ // Barrier
+ // ...
+ // Loads/Stores
+ // Barrier (LastBarrier)
+ // Loads/Stores (Chunks.back())
+ //
+ // Since loads/stores must not be reordered across barrier instructions, and
+ // the order of barrier instructions must be preserved, add the following
+ // loop-carried dependences:
+ //
+ // - LastBarrier -> Loads/Stores in Chunks.front()
+ // - Loads/Stores in Chunks.back() -> FirstBarrier
+ // - LastBarrier -> FirstBarrier (if they are different)
+ //
+ if (FirstBarrier) {
+ assert(LastBarrier && "Both barriers should be set.");
+ for (const SUnitWithMemInfo &Dst : Chunks.front().Loads)
+ setLoopCarriedDep(LastBarrier, Dst.SU);
+ for (const SUnitWithMemInfo &Dst : Chunks.front().Stores)
+ setLoopCarriedDep(LastBarrier, Dst.SU);
+
+ for (const SUnitWithMemInfo &Src : Chunks.back().Loads)
+ setLoopCarriedDep(Src.SU, FirstBarrier);
+ for (const SUnitWithMemInfo &Src : Chunks.back().Stores)
+ setLoopCarriedDep(Src.SU, FirstBarrier);
+
+ if (FirstBarrier != LastBarrier)
+ setLoopCarriedDep(LastBarrier, FirstBarrier);
+ }
}
/// Add a chain edge between a load and store if the store can be an
diff --git a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
index bcc6a3ea9b285..214f7e245030a 100644
--- a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
+++ b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
@@ -5,16 +5,24 @@
# floating-point exception, and there is an instruction for barrier event. In
# this case the order of them must not change.
#
+# SU(2): May raise FP exception
+# SU(3): May raise FP exception
+# SU(4): Store
+# SU(5): Barrier
+# SU(7): Barrier
+#
# FIXME: Currently the following dependencies are missed.
#
# Loop carried edges from SU(7)
# Order
# SU(2)
# SU(3)
-# SU(4)
-# SU(5)
# CHECK: ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: Loop carried edges from SU(7)
+# CHECK-NEXT: Order
+# CHECK-NEXT: SU(4)
+# CHECK-NEXT: SU(5)
# CHECK-NEXT: ===== Loop Carried Edges End =====
--- |
diff --git a/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir b/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir
index 4281d15377141..8fe3b3d83aa94 100644
--- a/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir
@@ -19,23 +19,33 @@
# }
# ```
#
-# FIXME: Currently the following dependencies are missed.
-# Loop carried edges from SU(16)
-# Order
-# SU(6)
-# SU(8)
-# SU(10)
-# SU(11)
-# Loop carried edges from SU(17)
-# Order
-# SU(10)
-# SU(11)
-# Loop carried edges from SU(19)
-# Order
-# SU(10)
-# SU(11)
+# SU(6): Load
+# SU(8): Store
+# SU(10): Store
+# SU(11): Barrier
+# SU(16): Barrier
+# SU(17): Load
+# SU(19): Load
+#
+# As the order between load/store and barrier must be preserved, the following
+# loop-carried dependnecies need to be added:
+# - SU(16) -> SU(6), SU(8), SU(10)
+# - SU(17), SU(19) -> SU(11)
+# - SU(16) -> SU(11) (barrier to barrier)
# CHECK: ===== Loop Carried Edges Begin =====
+# CHECK-NEXT: Loop carried edges from SU(16)
+# CHECK-NEXT: Order
+# CHECK-NEXT: SU(6)
+# CHECK-NEXT: SU(8)
+# CHECK-NEXT: SU(10)
+# CHECK-NEXT: SU(11)
+# CHECK-NEXT: Loop carried edges from SU(17)
+# CHECK-NEXT: Order
+# CHECK-NEXT: SU(11)
+# CHECK-NEXT: Loop carried edges from SU(19)
+# CHECK-NEXT: Order
+# CHECK-NEXT: SU(11)
# CHECK-NEXT: ===== Loop Carried Edges End =====
--- |
More information about the llvm-branch-commits
mailing list