[llvm-branch-commits] [llvm] [MachinePipeliner] Add loop-carried dependences for FPExceptions (PR #174392)
Ryotaro Kasuga via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jan 5 04:23:04 PST 2026
https://github.com/kasuga-fj updated https://github.com/llvm/llvm-project/pull/174392
>From 533b9a6fa6998bb9b56fdc4ecf6bb531c1bbbaed Mon Sep 17 00:00:00 2001
From: Ryotaro Kasuga <kasuga.ryotaro at fujitsu.com>
Date: Fri, 26 Dec 2025 07:21:07 +0000
Subject: [PATCH] [MachinePipeliner] Add loop-carried dependences for
FPExceptions
---
llvm/lib/CodeGen/MachinePipeliner.cpp | 77 +++++++++++--------
.../sms-loop-carried-fp-exceptions1.mir | 9 +--
2 files changed, 47 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 4c8ef0aaa3d66..08138ea140db7 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -290,10 +290,12 @@ class LoopCarriedOrderDepsTracker {
InstrTag getTag() const { return InstrTag(getInt()); }
};
- /// Holds loads and stores with memory related information.
- struct LoadStoreChunk {
+ /// Holds instructions that may form loop-carried order-dependencies, but not
+ /// global barriers.
+ struct NoBarrierInstsChunk {
SmallVector<SUnitWithMemInfo, 4> Loads;
SmallVector<SUnitWithMemInfo, 4> Stores;
+ SmallVector<SUnitWithMemInfo, 1> FPExceptions;
void append(SUnit *SU);
};
@@ -340,8 +342,8 @@ class LoopCarriedOrderDepsTracker {
/// Tags to \p SU if the instruction may affect the order-dependencies.
std::optional<InstrTag> getInstrTag(SUnit *SU) const;
- void addLoopCarriedDepenenciesForChunks(const LoadStoreChunk &From,
- const LoadStoreChunk &To);
+ void addLoopCarriedDepenenciesForChunks(const NoBarrierInstsChunk &From,
+ const NoBarrierInstsChunk &To);
/// Add a loop-carried order dependency between \p Src and \p Dst if we
/// cannot prove they are independent.
@@ -1091,11 +1093,16 @@ static bool hasLoopCarriedMemDep(const SUnitWithMemInfo &Src,
return false;
}
-void LoopCarriedOrderDepsTracker::LoadStoreChunk::append(SUnit *SU) {
+void LoopCarriedOrderDepsTracker::NoBarrierInstsChunk::append(SUnit *SU) {
const MachineInstr *MI = SU->getInstr();
- if (!MI->mayLoadOrStore())
- return;
- (MI->mayStore() ? Stores : Loads).emplace_back(SU);
+ if (MI->mayStore())
+ Stores.emplace_back(SU);
+ else if (MI->mayLoad())
+ Loads.emplace_back(SU);
+ else if (MI->mayRaiseFPException())
+ FPExceptions.emplace_back(SU);
+ else
+ llvm_unreachable("Unexpected instruction type.");
}
LoopCarriedOrderDepsTracker::LoopCarriedOrderDepsTracker(
@@ -1145,7 +1152,7 @@ void LoopCarriedOrderDepsTracker::addDependenciesBetweenSUs(
}
void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks(
- const LoadStoreChunk &From, const LoadStoreChunk &To) {
+ const NoBarrierInstsChunk &From, const NoBarrierInstsChunk &To) {
// Add load-to-store dependencies (WAR).
for (const SUnitWithMemInfo &Src : From.Loads)
for (const SUnitWithMemInfo &Dst : To.Stores)
@@ -1163,7 +1170,7 @@ void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks(
}
void LoopCarriedOrderDepsTracker::computeDependenciesAux() {
- SmallVector<LoadStoreChunk, 2> Chunks(1);
+ SmallVector<NoBarrierInstsChunk, 2> Chunks(1);
SUnit *FirstBarrier = nullptr;
SUnit *LastBarrier = nullptr;
for (const auto &TSU : TaggedSUnits) {
@@ -1177,10 +1184,8 @@ void LoopCarriedOrderDepsTracker::computeDependenciesAux() {
Chunks.emplace_back();
break;
case InstrTag::LoadOrStore:
- Chunks.back().append(SU);
- break;
case InstrTag::FPExceptions:
- // TODO: Handle this properly.
+ Chunks.back().append(SU);
break;
}
}
@@ -1188,47 +1193,55 @@ void LoopCarriedOrderDepsTracker::computeDependenciesAux() {
// Add dependencies between memory operations. If there are one or more
// barrier events between two memory instructions, we don't add a
// loop-carried dependence for them.
- for (const LoadStoreChunk &Chunk : Chunks)
+ for (const NoBarrierInstsChunk &Chunk : Chunks)
addLoopCarriedDepenenciesForChunks(Chunk, Chunk);
- // There is no barrier instruction between load/store instructions in the same
- // LoadStoreChunk. If there are one or more barrier instructions, the
- // instructions sequence is as follows:
+ // There is no barrier instruction between load/store/fp-exception
+ // instructions in the same chunk. If there are one or more barrier
+ // instructions, the instructions sequence is as follows:
//
- // Loads/Stores (Chunks.front())
+ // Loads/Stores/FPExceptions (Chunks.front())
// Barrier (FirstBarrier)
- // Loads/Stores
+ // Loads/Stores/FPExceptions
// Barrier
// ...
- // Loads/Stores
+ // Loads/Stores/FPExceptions
// Barrier (LastBarrier)
- // Loads/Stores (Chunks.back())
+ // Loads/Stores/FPExceptions (Chunks.back())
//
- // Since loads/stores must not be reordered across barrier instructions, and
- // the order of barrier instructions must be preserved, add the following
- // loop-carried dependences:
+ // Since loads/stores/fp-exceptions must not be reordered across barrier
+ // instructions, and the order of barrier instructions must be preserved, add
+ // the following loop-carried dependences:
//
- // Loads/Stores (Chunks.front()) <-----+
- // +--> Barrier (FirstBarrier) <---------+ |
- // | Loads/Stores | |
- // | Barrier | |
- // | ... | |
- // | Loads/Stores | |
- // | Barrier (LastBarrier) -----------+--+
- // +--- Loads/Stores (Chunks.back())
+ // Loads/Stores/FPExceptions (Chunks.front()) <-----+
+ // +--> Barrier (FirstBarrier) <----------------------+ |
+ // | Loads/Stores/FPExceptions | |
+ // | Barrier | |
+ // | ... | |
+ // | Loads/Stores/FPExceptions | |
+ // | Barrier (LastBarrier) ------------------------+--+
+ // +--- Loads/Stores/FPExceptions (Chunks.back())
//
if (FirstBarrier) {
assert(LastBarrier && "Both barriers should be set.");
+
+ // LastBarrier -> Loads/Stores/FPExceptions in Chunks.front()
for (const SUnitWithMemInfo &Dst : Chunks.front().Loads)
setLoopCarriedDep(LastBarrier, Dst.SU);
for (const SUnitWithMemInfo &Dst : Chunks.front().Stores)
setLoopCarriedDep(LastBarrier, Dst.SU);
+ for (const SUnitWithMemInfo &Dst : Chunks.front().FPExceptions)
+ setLoopCarriedDep(LastBarrier, Dst.SU);
+ // Loads/Stores/FPExceptions in Chunks.back() -> FirstBarrier
for (const SUnitWithMemInfo &Src : Chunks.back().Loads)
setLoopCarriedDep(Src.SU, FirstBarrier);
for (const SUnitWithMemInfo &Src : Chunks.back().Stores)
setLoopCarriedDep(Src.SU, FirstBarrier);
+ for (const SUnitWithMemInfo &Src : Chunks.back().FPExceptions)
+ setLoopCarriedDep(Src.SU, FirstBarrier);
+ // LastBarrier -> FirstBarrier (if they are different)
if (FirstBarrier != LastBarrier)
setLoopCarriedDep(LastBarrier, FirstBarrier);
}
diff --git a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
index 214f7e245030a..6fddf549a0b48 100644
--- a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
+++ b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir
@@ -10,17 +10,12 @@
# SU(4): Store
# SU(5): Barrier
# SU(7): Barrier
-#
-# FIXME: Currently the following dependencies are missed.
-#
-# Loop carried edges from SU(7)
-# Order
-# SU(2)
-# SU(3)
# CHECK: ===== Loop Carried Edges Begin =====
# CHECK-NEXT: Loop carried edges from SU(7)
# CHECK-NEXT: Order
+# CHECK-NEXT: SU(2)
+# CHECK-NEXT: SU(3)
# CHECK-NEXT: SU(4)
# CHECK-NEXT: SU(5)
# CHECK-NEXT: ===== Loop Carried Edges End =====
More information about the llvm-branch-commits
mailing list