[llvm] [ORC] Replace ORC's baked-in dependence tracking with WaitingOnGraph. (PR #162987)
Lang Hames via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 11 02:55:58 PDT 2025
https://github.com/lhames created https://github.com/llvm/llvm-project/pull/162987
WaitingOnGraph tracks waiting-on relationships between nodes (intended to represent symbols in an ORC program) in order to identify nodes that are *Ready* (i.e. are not waiting on any other nodes) or have *Failed* (are waiting on some node that cannot be produced).
WaitingOnGraph replaces ORC's baked-in data structures that were tracking the same information (EmissionDepUnit, EmissionDepUnitInfo, ...). Isolating this information in a separate data structure simplifies the code, allows us to unit test it, and simplifies performance testing.
The WaitingOnGraph uses several techniques to improve performance relative to the old data structures, including symbol coalescing ("SuperNodes") and symbol keys that don't perform unnecessary reference counting (NonOwningSymbolStringPtr).
This commit includes unit tests for common dependence-tracking issues that have led to ORC bugs in the past.
>From 3b9eb34f72681d752c84347997b9c53ddd9a80dd Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames at gmail.com>
Date: Fri, 10 Oct 2025 14:47:57 +1100
Subject: [PATCH] [ORC] Replace ORC's baked-in dependence tracking with
WaitingOnGraph.
WaitingOnGraph tracks waiting-on relationships between nodes (intended to
represent symbols in an ORC program) in order to identify nodes that are
*Ready* (i.e. are not waiting on any other nodes) or have *Failed* (are
waiting on some node that cannot be produced).
WaitingOnGraph replaces ORC's baked-in data structures that were tracking the
same information (EmissionDepUnit, EmissionDepUnitInfo, ...). Isolating this
information in a separate data structure simplifies the code, allows us to
unit test it, and simplifies performance testing.
The WaitingOnGraph uses several techniques to improve performance relative to
the old data structures, including symbol coalescing ("SuperNodes") and symbol
keys that don't perform unnecessary reference counting
(NonOwningSymbolStringPtr).
This commit includes unit tests for common dependence-tracking issues that have
led to ORC bugs in the past.
---
llvm/include/llvm/ExecutionEngine/Orc/Core.h | 66 +-
.../ExecutionEngine/Orc/SymbolStringPool.h | 8 +-
.../llvm/ExecutionEngine/Orc/WaitingOnGraph.h | 622 ++++++++++++
llvm/lib/ExecutionEngine/Orc/Core.cpp | 899 ++++--------------
.../ExecutionEngine/Orc/SimpleRemoteEPC.cpp | 2 +-
.../AArch64/MachO_ptrauth_dependencies.s | 2 +-
.../x86-64/LocalDependencyPropagation.s | 3 +-
.../ExecutionEngine/Orc/CMakeLists.txt | 1 +
.../Orc/WaitingOnGraphTest.cpp | 553 +++++++++++
9 files changed, 1417 insertions(+), 739 deletions(-)
create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h
create mode 100644 llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index f407b56817fc3..8613ddd8e3b11 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -26,6 +26,7 @@
#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
#include "llvm/ExecutionEngine/Orc/TaskDispatch.h"
+#include "llvm/ExecutionEngine/Orc/WaitingOnGraph.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ExtensibleRTTI.h"
@@ -49,6 +50,9 @@ class InProgressLookupState;
enum class SymbolState : uint8_t;
+using WaitingOnGraph =
+ detail::WaitingOnGraph<JITDylib *, NonOwningSymbolStringPtr>;
+
using ResourceTrackerSP = IntrusiveRefCntPtr<ResourceTracker>;
using JITDylibSP = IntrusiveRefCntPtr<JITDylib>;
@@ -1131,20 +1135,6 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
using UnmaterializedInfosList =
std::vector<std::shared_ptr<UnmaterializedInfo>>;
- struct EmissionDepUnit {
- EmissionDepUnit(JITDylib &JD) : JD(&JD) {}
-
- JITDylib *JD = nullptr;
- DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> Symbols;
- DenseMap<JITDylib *, DenseSet<NonOwningSymbolStringPtr>> Dependencies;
- };
-
- struct EmissionDepUnitInfo {
- std::shared_ptr<EmissionDepUnit> EDU;
- DenseSet<EmissionDepUnit *> IntraEmitUsers;
- DenseMap<JITDylib *, DenseSet<NonOwningSymbolStringPtr>> NewDeps;
- };
-
// Information about not-yet-ready symbol.
// * DefiningEDU will point to the EmissionDepUnit that defines the symbol.
// * DependantEDUs will hold pointers to any EmissionDepUnits currently
@@ -1154,9 +1144,6 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
struct MaterializingInfo {
friend class ExecutionSession;
- std::shared_ptr<EmissionDepUnit> DefiningEDU;
- DenseSet<EmissionDepUnit *> DependantEDUs;
-
LLVM_ABI void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q);
LLVM_ABI void removeQuery(const AsynchronousSymbolQuery &Q);
LLVM_ABI AsynchronousSymbolQueryList
@@ -1778,30 +1765,26 @@ class ExecutionSession {
LLVM_ABI Error OL_notifyResolved(MaterializationResponsibility &MR,
const SymbolMap &Symbols);
- using EDUInfosMap =
- DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo>;
-
- template <typename HandleNewDepFn>
- void propagateExtraEmitDeps(std::deque<JITDylib::EmissionDepUnit *> Worklist,
- EDUInfosMap &EDUInfos,
- HandleNewDepFn HandleNewDep);
- EDUInfosMap simplifyDepGroups(MaterializationResponsibility &MR,
- ArrayRef<SymbolDependenceGroup> EmittedDeps);
- void IL_makeEDUReady(std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries);
- void IL_makeEDUEmitted(std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries);
- bool IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU, JITDylib &DepJD,
- NonOwningSymbolStringPtr DepSym,
- EDUInfosMap &EDUInfos);
-
- static Error makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &ClosedJD);
- static Error makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &BadJD, SymbolNameSet BadDeps);
-
- Expected<JITDylib::AsynchronousSymbolQuerySet>
- IL_emit(MaterializationResponsibility &MR, EDUInfosMap EDUInfos);
+ // FIXME: We should be able to derive FailedSymsForQuery from each query once
+ // we fix how the detach operation works.
+ struct EmitQueries {
+ JITDylib::AsynchronousSymbolQuerySet Updated;
+ JITDylib::AsynchronousSymbolQuerySet Failed;
+ DenseMap<AsynchronousSymbolQuery *, std::shared_ptr<SymbolDependenceMap>>
+ FailedSymsForQuery;
+ };
+
+ WaitingOnGraph::ExternalState
+ IL_getSymbolState(JITDylib *JD, NonOwningSymbolStringPtr Name);
+
+ template <typename UpdateSymbolFn, typename UpdateQueryFn>
+ void IL_collectQueries(JITDylib::AsynchronousSymbolQuerySet &Qs,
+ WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
+ UpdateSymbolFn &&UpdateSymbol,
+ UpdateQueryFn &&UpdateQuery);
+
+ Expected<EmitQueries> IL_emit(MaterializationResponsibility &MR,
+ WaitingOnGraph::SimplifyResult SR);
LLVM_ABI Error OL_notifyEmitted(MaterializationResponsibility &MR,
ArrayRef<SymbolDependenceGroup> EmittedDeps);
@@ -1830,6 +1813,7 @@ class ExecutionSession {
std::vector<ResourceManager *> ResourceManagers;
std::vector<JITDylibSP> JDs;
+ WaitingOnGraph G;
// FIXME: Remove this (and runOutstandingMUs) once the linking layer works
// with callbacks from asynchronous queries.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
index ed6ea961bccfe..5b65a175527d7 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
@@ -14,6 +14,7 @@
#define LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/Compiler.h"
#include <atomic>
@@ -71,6 +72,7 @@ class SymbolStringPool {
/// from nullptr to enable comparison with these values.
class SymbolStringPtrBase {
friend class SymbolStringPool;
+ friend class SymbolStringPoolEntryUnsafe;
friend struct DenseMapInfo<SymbolStringPtr>;
friend struct DenseMapInfo<NonOwningSymbolStringPtr>;
@@ -204,7 +206,7 @@ class SymbolStringPoolEntryUnsafe {
SymbolStringPoolEntryUnsafe(PoolEntry *E) : E(E) {}
/// Create an unsafe pool entry ref without changing the ref-count.
- static SymbolStringPoolEntryUnsafe from(const SymbolStringPtr &S) {
+ static SymbolStringPoolEntryUnsafe from(const SymbolStringPtrBase &S) {
return S.S;
}
@@ -318,6 +320,10 @@ SymbolStringPool::getRefCount(const SymbolStringPtrBase &S) const {
LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
const SymbolStringPtrBase &Sym);
+inline hash_code hash_value(const orc::NonOwningSymbolStringPtr &S) {
+ return hash_value(orc::SymbolStringPoolEntryUnsafe::from(S).rawPtr());
+}
+
} // end namespace orc
template <>
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h
new file mode 100644
index 0000000000000..a5b533351d4d0
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h
@@ -0,0 +1,622 @@
+//===------ WaitingOnGraph.h - ORC symbol dependence graph ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines WaitingOnGraph and related utilities.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H
+#define LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <algorithm>
+
+namespace llvm::orc::detail {
+
+class WaitingOnGraphTest;
+
+/// WaitingOnGraph class template.
+///
+/// This type is intended to provide efficient dependence tracking for Symbols
+/// in an ORC program.
+///
+/// WaitingOnGraph models a directed graph with four partitions:
+/// 1. Not-yet-emitted nodes: Nodes identified as waited-on in an emit
+/// operation.
+/// 2. Emitted nodes: Nodes emitted and waiting on some non-empty set of
+/// other nodes.
+/// 3. Ready nodes: Nodes emitted and not waiting on any other nodes
+/// (either because they weren't waiting on any nodes when they were
+/// emitted, or because all transitively waited-on nodes have since
+/// been emitted).
+/// 4. Failed nodes: Nodes that have been marked as failed-to-emit, and
+/// nodes that were found to transitively wait-on some failed node.
+///
+/// Nodes are added to the graph by *emit* and *fail* operations.
+///
+/// The *emit* operation takes a bipartite *local dependence graph* as an
+/// argument and returns...
+/// a. the set of nodes (both existing and newly added from the local
+/// dependence graph) whose waiting-on set is the empty set, and...
+/// b. the set of newly added nodes that are found to depend on failed
+/// nodes.
+///
+/// The *fail* operation takes a set of failed nodes and returns the set of
+/// Emitted nodes that were waiting on the failed nodes.
+///
+/// The concrete representation adopts several approaches for efficiency:
+///
+/// 1. Only *Emitted* and *Not-yet-emitted* nodes are represented explicitly.
+/// *Ready* and *Failed* nodes are represented by the values returned by the
+/// GetExternalStateFn argument to *emit*.
+///
+/// 2. Labels are (*Container*, *Element*) pairs that are intended to represent
+/// ORC symbols (ORC uses types Container = JITDylib,
+/// Element = NonOwningSymbolStringPtr). The internal representation of the
+/// graph is optimized on the assumption that there are many more Elements
+/// (symbol names) than Containers (JITDylibs) used to construct the labels.
+/// (Consider for example the common case where most JIT'd code is placed in
+/// a single "main" JITDylib).
+///
+/// 3. The data structure stores *SuperNodes* which have multiple labels. This
+/// reduces the number of nodes and edges in the graph in the common case
+/// where many JIT symbols have the same set of dependencies. SuperNodes are
+/// coalesced when their dependence sets become equal.
+///
+/// 4. The *simplify* method can be applied to an initial *local dependence
+/// graph* (as a list of SuperNodes) to eliminate any internal dependence
+/// relationships that would have to be propagated internally by *emit*.
+/// Access to the WaitingOnGraph is assumed to be guarded by a mutex (ORC
+/// will access it from multiple threads) so this allows some pre-processing
+/// to be performed outside the mutex.
+template <typename ContainerIdT, typename ElementIdT> class WaitingOnGraph {
+ friend class WaitingOnGraphTest;
+
+public:
+ using ContainerId = ContainerIdT;
+ using ElementId = ElementIdT;
+ using ElementSet = DenseSet<ElementId>;
+ using ContainerElementsMap = DenseMap<ContainerId, ElementSet>;
+
+ class SuperNode {
+ friend class WaitingOnGraph;
+ friend class WaitingOnGraphTest;
+
+ public:
+ SuperNode(ContainerElementsMap Defs, ContainerElementsMap Deps)
+ : Defs(std::move(Defs)), Deps(std::move(Deps)) {}
+ ContainerElementsMap &defs() { return Defs; }
+ const ContainerElementsMap &defs() const { return Defs; }
+ ContainerElementsMap &deps() { return Deps; }
+ const ContainerElementsMap &deps() const { return Deps; }
+
+ private:
+ ContainerElementsMap Defs;
+ ContainerElementsMap Deps;
+ };
+
+private:
+ using ElemToSuperNodeMap =
+ DenseMap<ContainerId, DenseMap<ElementId, SuperNode *>>;
+
+ using SuperNodeDepsMap = DenseMap<SuperNode *, DenseSet<SuperNode *>>;
+
+ class Coalescer {
+ public:
+ std::unique_ptr<SuperNode> addOrCreateSuperNode(ContainerElementsMap Defs,
+ ContainerElementsMap Deps) {
+ auto H = getHash(Deps);
+ if (auto *ExistingSN = findCanonicalSuperNode(H, Deps)) {
+ for (auto &[Container, Elems] : Defs) {
+ auto &DstCElems = ExistingSN->Defs[Container];
+ [[maybe_unused]] size_t ExpectedSize =
+ DstCElems.size() + Elems.size();
+ DstCElems.insert(Elems.begin(), Elems.end());
+ assert(DstCElems.size() == ExpectedSize);
+ }
+ return nullptr;
+ }
+
+ auto NewSN =
+ std::make_unique<SuperNode>(std::move(Defs), std::move(Deps));
+ CanonicalSNs[H].push_back(NewSN.get());
+ return NewSN;
+ }
+
+ void coalesce(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ ElemToSuperNodeMap &ElemToSN) {
+ for (size_t I = 0; I != SNs.size();) {
+ auto &SN = SNs[I];
+ auto H = getHash(SN->Deps);
+ if (auto *CanonicalSN = findCanonicalSuperNode(H, SN->Deps)) {
+ for (auto &[Container, Elems] : SN->Defs) {
+ CanonicalSN->Defs[Container].insert(Elems.begin(), Elems.end());
+ auto &ContainerElemToSN = ElemToSN[Container];
+ for (auto &Elem : Elems)
+ ContainerElemToSN[Elem] = CanonicalSN;
+ }
+ std::swap(SN, SNs.back());
+ SNs.pop_back();
+ } else {
+ CanonicalSNs[H].push_back(SN.get());
+ ++I;
+ }
+ }
+ }
+
+ template <typename Pred> void remove(Pred &&Remove) {
+ for (auto &[Hash, SNs] : CanonicalSNs) {
+ bool Found = false;
+ for (size_t I = 0; I != SNs.size(); ++I) {
+ if (Remove(SNs[I])) {
+ std::swap(SNs[I], SNs.back());
+ SNs.pop_back();
+ Found = true;
+ break;
+ }
+ }
+ if (Found) {
+ if (SNs.empty())
+ CanonicalSNs.erase(Hash);
+ break;
+ }
+ }
+ }
+
+ private:
+ hash_code getHash(const ContainerElementsMap &M) {
+ SmallVector<ContainerId> SortedContainers;
+ SortedContainers.reserve(M.size());
+ for (auto &[Container, Elems] : M)
+ SortedContainers.push_back(Container);
+ llvm::sort(SortedContainers);
+ hash_code Hash(0);
+ for (auto &Container : SortedContainers) {
+ auto &ContainerElems = M.at(Container);
+ SmallVector<ElementId> SortedElems(ContainerElems.begin(),
+ ContainerElems.end());
+ llvm::sort(SortedElems);
+ Hash = hash_combine(
+ Hash, Container,
+ hash_combine_range(SortedElems.begin(), SortedElems.end()));
+ }
+ return Hash;
+ }
+
+ SuperNode *findCanonicalSuperNode(hash_code H,
+ const ContainerElementsMap &M) {
+ for (auto *SN : CanonicalSNs[H])
+ if (SN->Deps == M)
+ return SN;
+ return nullptr;
+ }
+
+ DenseMap<hash_code, SmallVector<SuperNode *>> CanonicalSNs;
+ };
+
+public:
+ /// Build SuperNodes from (definition-set, dependence-set) pairs.
+ ///
+ /// Coalesces definition-sets with identical dependence-sets.
+ class SuperNodeBuilder {
+ public:
+ void add(ContainerElementsMap Defs, ContainerElementsMap Deps) {
+ if (Defs.empty())
+ return;
+ // Remove any self-reference.
+ SmallVector<ContainerId> ToRemove;
+ for (auto &[Container, Elems] : Defs) {
+ assert(!Elems.empty() && "Defs for container must not be empty");
+ auto I = Deps.find(Container);
+ if (I == Deps.end())
+ continue;
+ auto &DepsForContainer = I->second;
+ for (auto &Elem : Elems)
+ DepsForContainer.erase(Elem);
+ if (DepsForContainer.empty())
+ ToRemove.push_back(Container);
+ }
+ for (auto &Container : ToRemove)
+ Deps.erase(Container);
+ if (auto SN = C.addOrCreateSuperNode(std::move(Defs), std::move(Deps)))
+ SNs.push_back(std::move(SN));
+ }
+ std::vector<std::unique_ptr<SuperNode>> takeSuperNodes() {
+ return std::move(SNs);
+ }
+
+ private:
+ Coalescer C;
+ std::vector<std::unique_ptr<SuperNode>> SNs;
+ };
+
+ class SimplifyResult {
+ friend class WaitingOnGraph;
+ friend class WaitingOnGraphTest;
+
+ public:
+ const std::vector<std::unique_ptr<SuperNode>> &superNodes() const {
+ return SNs;
+ }
+
+ private:
+ SimplifyResult(std::vector<std::unique_ptr<SuperNode>> SNs,
+ ElemToSuperNodeMap ElemToSN)
+ : SNs(std::move(SNs)), ElemToSN(std::move(ElemToSN)) {}
+ std::vector<std::unique_ptr<SuperNode>> SNs;
+ ElemToSuperNodeMap ElemToSN;
+ };
+
+ /// Preprocess a list of SuperNodes to remove all intra-SN dependencies.
+ static SimplifyResult simplify(std::vector<std::unique_ptr<SuperNode>> SNs) {
+ // Build ElemToSN map.
+ ElemToSuperNodeMap ElemToSN;
+ for (auto &SN : SNs) {
+ for (auto &[Container, Elements] : SN->Defs) {
+ auto &ContainerElemToSN = ElemToSN[Container];
+ for (auto &E : Elements)
+ ContainerElemToSN[E] = SN.get();
+ }
+ }
+
+ SuperNodeDepsMap SuperNodeDeps;
+ hoistDeps(SuperNodeDeps, SNs, ElemToSN);
+ propagateSuperNodeDeps(SuperNodeDeps);
+ sinkDeps(SNs, SuperNodeDeps);
+
+ // Pre-coalesce nodes.
+ Coalescer().coalesce(SNs, ElemToSN);
+
+ return {std::move(SNs), std::move(ElemToSN)};
+ }
+
+ struct EmitResult {
+ std::vector<std::unique_ptr<SuperNode>> Ready;
+ std::vector<std::unique_ptr<SuperNode>> Failed;
+ };
+
+ enum class ExternalState { None, Ready, Failed };
+
+ /// Add the given SuperNodes to the graph, returning any SuperNodes that
+ /// move to the Ready or Failed states as a result.
+ /// The GetExternalState function is used to represent SuperNodes that have
+ /// already become Ready or Failed (since such nodes are not explicitly
+ /// represented in the graph).
+ template <typename GetExternalStateFn>
+ EmitResult emit(SimplifyResult SR, GetExternalStateFn &&GetExternalState) {
+ auto NewSNs = std::move(SR.SNs);
+ auto ElemToNewSN = std::move(SR.ElemToSN);
+
+ // First process any dependencies on nodes with external state.
+ auto FailedSNs = processExternalDeps(NewSNs, GetExternalState);
+
+ // Collect the PendingSNs whose dep sets are about to be modified.
+ std::vector<std::unique_ptr<SuperNode>> ModifiedPendingSNs;
+ for (size_t I = 0; I != PendingSNs.size();) {
+ auto &SN = PendingSNs[I];
+ bool Remove = false;
+ for (auto &[Container, Elems] : SN->Deps) {
+ auto I = ElemToNewSN.find(Container);
+ if (I == ElemToNewSN.end())
+ continue;
+ for (auto Elem : Elems) {
+ if (I->second.contains(Elem)) {
+ Remove = true;
+ break;
+ }
+ }
+ if (Remove)
+ break;
+ }
+ if (Remove) {
+ ModifiedPendingSNs.push_back(std::move(SN));
+ std::swap(SN, PendingSNs.back());
+ PendingSNs.pop_back();
+ } else
+ ++I;
+ }
+
+ // Remove cycles from the graphs.
+ SuperNodeDepsMap SuperNodeDeps;
+ hoistDeps(SuperNodeDeps, ModifiedPendingSNs, ElemToNewSN);
+
+ CoalesceToPendingSNs.remove(
+ [&](SuperNode *SN) { return SuperNodeDeps.count(SN); });
+
+ hoistDeps(SuperNodeDeps, NewSNs, ElemToPendingSN);
+ propagateSuperNodeDeps(SuperNodeDeps);
+ sinkDeps(NewSNs, SuperNodeDeps);
+ sinkDeps(ModifiedPendingSNs, SuperNodeDeps);
+
+ // Process supernodes. Pending first, since we'll update PendingSNs when we
+ // incorporate NewSNs.
+ std::vector<std::unique_ptr<SuperNode>> ReadyNodes, FailedNodes;
+ processReadyOrFailed(ModifiedPendingSNs, ReadyNodes, FailedNodes,
+ SuperNodeDeps, ElemToPendingSN, FailedSNs);
+ processReadyOrFailed(NewSNs, ReadyNodes, FailedNodes, SuperNodeDeps,
+ ElemToNewSN, FailedSNs);
+
+ CoalesceToPendingSNs.coalesce(ModifiedPendingSNs, ElemToPendingSN);
+ CoalesceToPendingSNs.coalesce(NewSNs, ElemToPendingSN);
+
+ // Integrate remaining ModifiedPendingSNs and NewSNs into PendingSNs.
+ for (auto &SN : ModifiedPendingSNs)
+ PendingSNs.push_back(std::move(SN));
+
+ // Update ElemToPendingSN for the remaining elements.
+ for (auto &SN : NewSNs) {
+ for (auto &[Container, Elems] : SN->Defs) {
+ auto &Row = ElemToPendingSN[Container];
+ for (auto &Elem : Elems)
+ Row[Elem] = SN.get();
+ }
+ PendingSNs.push_back(std::move(SN));
+ }
+
+ return {std::move(ReadyNodes), std::move(FailedNodes)};
+ }
+
+ /// Identify the given symbols as Failed.
+ /// The elements of the Failed map will not be included in the returned
+ /// result, so clients should take whatever actions are needed to mark
+ /// this as failed in their external representation.
+ std::vector<std::unique_ptr<SuperNode>>
+ fail(const ContainerElementsMap &Failed) {
+ std::vector<std::unique_ptr<SuperNode>> FailedSNs;
+
+ for (size_t I = 0; I != PendingSNs.size();) {
+ auto &PendingSN = PendingSNs[I];
+ bool FailPendingSN = false;
+ for (auto &[Container, Elems] : PendingSN->Deps) {
+ if (FailPendingSN)
+ break;
+ auto I = Failed.find(Container);
+ if (I == Failed.end())
+ continue;
+ for (auto &Elem : Elems) {
+ if (I->second.count(Elem)) {
+ FailPendingSN = true;
+ break;
+ }
+ }
+ }
+ if (FailPendingSN) {
+ FailedSNs.push_back(std::move(PendingSN));
+ PendingSN = std::move(PendingSNs.back());
+ PendingSNs.pop_back();
+ } else
+ ++I;
+ }
+
+ for (auto &SN : FailedSNs) {
+ CoalesceToPendingSNs.remove(
+ [&](SuperNode *SNC) { return SNC == SN.get(); });
+ for (auto &[Container, Elems] : SN->Defs) {
+ assert(ElemToPendingSN.count(Container));
+ auto &CElems = ElemToPendingSN[Container];
+ for (auto &Elem : Elems)
+ CElems.erase(Elem);
+ if (CElems.empty())
+ ElemToPendingSN.erase(Container);
+ }
+ }
+
+ return FailedSNs;
+ }
+
+ bool validate(raw_ostream &Log) {
+ bool AllGood = true;
+ auto ErrLog = [&]() -> raw_ostream & {
+ AllGood = false;
+ return Log;
+ };
+
+ size_t DefCount = 0;
+ for (auto &PendingSN : PendingSNs) {
+ if (PendingSN->Deps.empty())
+ ErrLog() << "Pending SN " << PendingSN.get() << " has empty dep set.\n";
+ else {
+ bool BadElem = false;
+ for (auto &[Container, Elems] : PendingSN->Deps) {
+ auto I = ElemToPendingSN.find(Container);
+ if (I == ElemToPendingSN.end())
+ continue;
+ if (Elems.empty())
+ ErrLog() << "Pending SN " << PendingSN.get()
+ << " has dependence map entry for " << Container
+ << " with empty element set.\n";
+ for (auto &Elem : Elems) {
+ if (I->second.count(Elem)) {
+ ErrLog() << "Pending SN " << PendingSN.get()
+ << " has dependence on emitted element ( " << Container
+ << ", " << Elem << ")\n";
+ BadElem = true;
+ break;
+ }
+ }
+ if (BadElem)
+ break;
+ }
+ }
+
+ for (auto &[Container, Elems] : PendingSN->Defs) {
+ if (Elems.empty())
+ ErrLog() << "Pending SN " << PendingSN.get()
+ << " has def map entry for " << Container
+ << " with empty element set.\n";
+ DefCount += Elems.size();
+ auto I = ElemToPendingSN.find(Container);
+ if (I == ElemToPendingSN.end())
+ ErrLog() << "Pending SN " << PendingSN.get() << " has "
+ << Elems.size() << " defs in container " << Container
+ << " not covered by ElemsToPendingSN.\n";
+ else {
+ for (auto &Elem : Elems) {
+ auto J = I->second.find(Elem);
+ if (J == I->second.end())
+ ErrLog() << "Pending SN " << PendingSN.get() << " has element ("
+ << Container << ", " << Elem
+ << ") not covered by ElemsToPendingSN.\n";
+ else if (J->second != PendingSN.get())
+ ErrLog() << "ElemToPendingSN value invalid for (" << Container
+ << ", " << Elem << ")\n";
+ }
+ }
+ }
+ }
+
+ size_t DefCount2 = 0;
+ for (auto &[Container, Elems] : ElemToPendingSN)
+ DefCount2 += Elems.size();
+
+ assert(DefCount2 >= DefCount);
+ if (DefCount2 != DefCount)
+ ErrLog() << "ElemToPendingSN contains extra elements.\n";
+
+ return AllGood;
+ }
+
+private:
+ // Replace individual dependencies with supernode dependencies.
+ //
+ // For all dependencies in SNs, if the corresponding node is defined in
+ // ElemToSN then remove the individual dependency and add the record the
+ // dependency on the corresponding supernode in SuperNodeDeps.
+ static void hoistDeps(SuperNodeDepsMap &SuperNodeDeps,
+ std::vector<std::unique_ptr<SuperNode>> &SNs,
+ ElemToSuperNodeMap &ElemToSN) {
+ for (auto &SN : SNs) {
+ auto &SNDeps = SuperNodeDeps[SN.get()];
+ for (auto &[DefContainer, DefElems] : ElemToSN) {
+ auto I = SN->Deps.find(DefContainer);
+ if (I == SN->Deps.end())
+ continue;
+ for (auto &[DefElem, DefSN] : DefElems)
+ if (I->second.erase(DefElem))
+ SNDeps.insert(DefSN);
+ if (I->second.empty())
+ SN->Deps.erase(I);
+ }
+ }
+ }
+
+ // Compute transitive closure of deps for each node.
+ static void propagateSuperNodeDeps(SuperNodeDepsMap &SuperNodeDeps) {
+ for (auto &[SN, Deps] : SuperNodeDeps) {
+ DenseSet<SuperNode *> Reachable({SN});
+ SmallVector<SuperNode *> Worklist(Deps.begin(), Deps.end());
+
+ while (!Worklist.empty()) {
+ auto *DepSN = Worklist.pop_back_val();
+ if (!Reachable.insert(DepSN).second)
+ continue;
+ auto I = SuperNodeDeps.find(DepSN);
+ if (I == SuperNodeDeps.end())
+ continue;
+ for (auto *DepSNDep : I->second)
+ Worklist.push_back(DepSNDep);
+ }
+
+ Deps = std::move(Reachable);
+ }
+ }
+
+ // Sink SuperNode dependencies back to dependencies on individual nodes.
+ static void sinkDeps(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ SuperNodeDepsMap &SuperNodeDeps) {
+ for (auto &SN : SNs) {
+ auto I = SuperNodeDeps.find(SN.get());
+ if (I == SuperNodeDeps.end())
+ continue;
+
+ for (auto *DepSN : I->second)
+ for (auto &[Container, Elems] : DepSN->Deps)
+ SN->Deps[Container].insert(Elems.begin(), Elems.end());
+ }
+ }
+
+ template <typename GetExternalStateFn>
+ static std::vector<SuperNode *>
+ processExternalDeps(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ GetExternalStateFn &GetExternalState) {
+ std::vector<SuperNode *> FailedSNs;
+ for (auto &SN : SNs) {
+ bool SNHasError = false;
+ SmallVector<ContainerId> ContainersToRemove;
+ for (auto &[Container, Elems] : SN->Deps) {
+ SmallVector<ElementId> ElemToRemove;
+ for (auto &Elem : Elems) {
+ switch (GetExternalState(Container, Elem)) {
+ case ExternalState::None:
+ break;
+ case ExternalState::Ready:
+ ElemToRemove.push_back(Elem);
+ break;
+ case ExternalState::Failed:
+ ElemToRemove.push_back(Elem);
+ SNHasError = true;
+ break;
+ }
+ }
+ for (auto &Elem : ElemToRemove)
+ Elems.erase(Elem);
+ if (Elems.empty())
+ ContainersToRemove.push_back(Container);
+ }
+ for (auto &Container : ContainersToRemove)
+ SN->Deps.erase(Container);
+ if (SNHasError)
+ FailedSNs.push_back(SN.get());
+ }
+
+ return FailedSNs;
+ }
+
+ void processReadyOrFailed(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ std::vector<std::unique_ptr<SuperNode>> &Ready,
+ std::vector<std::unique_ptr<SuperNode>> &Failed,
+ SuperNodeDepsMap &SuperNodeDeps,
+ ElemToSuperNodeMap &ElemToSNs,
+ std::vector<SuperNode *> FailedSNs) {
+ for (size_t I = 0; I != SNs.size();) {
+ auto &SN = SNs[I];
+
+ bool SNFailed = false;
+ assert(SuperNodeDeps.count(SN.get()));
+ auto &SNSuperNodeDeps = SuperNodeDeps[SN.get()];
+ for (auto *FailedSN : FailedSNs) {
+ if (FailedSN == SN.get() || SNSuperNodeDeps.count(FailedSN)) {
+ SNFailed = true;
+ break;
+ }
+ }
+
+ bool SNReady = SN->Deps.empty();
+
+ if (SNReady || SNFailed) {
+ auto &NodeList = SNFailed ? Failed : Ready;
+ NodeList.push_back(std::move(SN));
+ std::swap(SN, SNs.back());
+ SNs.pop_back();
+ } else
+ ++I;
+ }
+ }
+
+ std::vector<std::unique_ptr<SuperNode>> PendingSNs;
+ ElemToSuperNodeMap ElemToPendingSN;
+ Coalescer CoalesceToPendingSNs;
+};
+
+} // namespace llvm::orc::detail
+
+#endif // LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index f47b7ecdcc7bb..0354d9ab57880 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -1173,39 +1173,7 @@ void JITDylib::dump(raw_ostream &OS) {
<< " pending queries: { ";
for (const auto &Q : KV.second.pendingQueries())
OS << Q.get() << " (" << Q->getRequiredState() << ") ";
- OS << "}\n Defining EDU: ";
- if (KV.second.DefiningEDU) {
- OS << KV.second.DefiningEDU.get() << " { ";
- for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
- OS << Name << " ";
- OS << "}\n";
- OS << " Dependencies:\n";
- if (!KV.second.DefiningEDU->Dependencies.empty()) {
- for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
- OS << " " << DepJD->getName() << ": [ ";
- for (auto &Dep : Deps)
- OS << Dep << " ";
- OS << "]\n";
- }
- } else
- OS << " none\n";
- } else
- OS << "none\n";
- OS << " Dependant EDUs:\n";
- if (!KV.second.DependantEDUs.empty()) {
- for (auto &DependantEDU : KV.second.DependantEDUs) {
- OS << " " << DependantEDU << ": "
- << DependantEDU->JD->getName() << " { ";
- for (auto &[Name, Flags] : DependantEDU->Symbols)
- OS << Name << " ";
- OS << "}\n";
- }
- } else
- OS << " none\n";
- assert((Symbols[KV.first].getState() != SymbolState::Ready ||
- (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
- !KV.second.DependantEDUs.empty())) &&
- "Stale materializing info entry");
+ OS << "}\n";
}
});
}
@@ -2917,359 +2885,64 @@ Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
return MR.JD.resolve(MR, Symbols);
}
-template <typename HandleNewDepFn>
-void ExecutionSession::propagateExtraEmitDeps(
- std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
- HandleNewDepFn HandleNewDep) {
-
- // Iterate to a fixed-point to propagate extra-emit dependencies through the
- // EDU graph.
- while (!Worklist.empty()) {
- auto &EDU = *Worklist.front();
- Worklist.pop_front();
-
- assert(EDUInfos.count(&EDU) && "No info entry for EDU");
- auto &EDUInfo = EDUInfos[&EDU];
-
- // Propagate new dependencies to users.
- for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
-
- // UserEDUInfo only present if UserEDU has its own users.
- JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
- {
- auto UserEDUInfoItr = EDUInfos.find(UserEDU);
- if (UserEDUInfoItr != EDUInfos.end())
- UserEDUInfo = &UserEDUInfoItr->second;
- }
-
- for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
- auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
- DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
- for (auto Dep : Deps) {
- if (UserEDUDepsForJD.insert(Dep).second) {
- HandleNewDep(*UserEDU, *DepJD, Dep);
- if (UserEDUInfo) {
- if (!UserEDUNewDepsForJD) {
- // If UserEDU has no new deps then it's not in the worklist
- // yet, so add it.
- if (UserEDUInfo->NewDeps.empty())
- Worklist.push_back(UserEDU);
- UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
- }
- // Add (DepJD, Dep) to NewDeps.
- UserEDUNewDepsForJD->insert(Dep);
- }
- }
- }
- }
- }
-
- EDUInfo.NewDeps.clear();
- }
-}
-
-// Note: This method modifies the emitted set.
-ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
- MaterializationResponsibility &MR,
- ArrayRef<SymbolDependenceGroup> EmittedDeps) {
-
- auto &TargetJD = MR.getTargetJITDylib();
-
- // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
- // Symbol -> EmissionDepUnit mappings.
- DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
- EDUInfos.reserve(EmittedDeps.size());
- DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
- for (auto &DG : EmittedDeps) {
- assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
-
- // Skip empty EDUs.
- if (DG.Dependencies.empty())
- continue;
-
- auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
- auto &EDUInfo = EDUInfos[TmpEDU.get()];
- EDUInfo.EDU = std::move(TmpEDU);
- for (const auto &Symbol : DG.Symbols) {
- NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
- assert(!EDUForSymbol.count(NonOwningSymbol) &&
- "Symbol should not appear in more than one SymbolDependenceGroup");
- assert(MR.getSymbols().count(Symbol) &&
- "Symbol in DepGroups not in the emitted set");
- auto NewlyEmittedItr = MR.getSymbols().find(Symbol);
- EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
- EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
- }
- }
-
- // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
- {
- DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
- for (auto &[Sym, Flags] : MR.getSymbols()) {
- if (!EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)))
- ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
- }
- if (!ResidualSymbolFlags.empty()) {
- auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
- ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
- auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
- ResidualEDUInfo.EDU = std::move(ResidualEDU);
-
- // If the residual EDU is the only one then bail out early.
- if (EDUInfos.size() == 1)
- return EDUInfos;
-
- // Otherwise add the residual EDU to the EDUForSymbol map.
- for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
- EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
- }
- }
-
-#ifndef NDEBUG
- assert(EDUForSymbol.size() == MR.getSymbols().size() &&
- "MR symbols not fully covered by EDUs?");
- for (auto &[Sym, Flags] : MR.getSymbols()) {
- assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
- "Sym in MR not covered by EDU");
- }
-#endif // NDEBUG
-
- // 3. Use the DepGroups array to build a graph of dependencies between
- // EmissionDepUnits in this finalization. We want to remove these
- // intra-finalization uses, propagating dependencies on symbols outside
- // this finalization. Add EDUs to the worklist.
- for (auto &DG : EmittedDeps) {
-
- // Skip SymbolDependenceGroups with no dependencies.
- if (DG.Dependencies.empty())
- continue;
-
- assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
- "No EDU for DG");
- auto &EDU =
- *EDUForSymbol.find(NonOwningSymbolStringPtr(*DG.Symbols.begin()))
- ->second;
-
- for (auto &[DepJD, Deps] : DG.Dependencies) {
- DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
-
- assert(!Deps.empty() && "Dependence set for DepJD is empty");
-
- if (DepJD != &TargetJD) {
- // DepJD is some other JITDylib.There can't be any intra-finalization
- // edges here, so just skip.
- for (auto &Dep : Deps)
- NewDepsForJD.insert(NonOwningSymbolStringPtr(Dep));
- } else {
- // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
- // skipping any and recording the intra-finalization use instead.
- for (auto &Dep : Deps) {
- NonOwningSymbolStringPtr NonOwningDep(Dep);
- auto I = EDUForSymbol.find(NonOwningDep);
- if (I == EDUForSymbol.end()) {
- if (!MR.getSymbols().count(Dep))
- NewDepsForJD.insert(NonOwningDep);
- continue;
- }
-
- if (I->second != &EDU)
- EDUInfos[I->second].IntraEmitUsers.insert(&EDU);
+WaitingOnGraph::ExternalState
+ExecutionSession::IL_getSymbolState(JITDylib *JD,
+ NonOwningSymbolStringPtr Name) {
+ if (JD->State != JITDylib::Open)
+ return WaitingOnGraph::ExternalState::Failed;
+
+ auto I = JD->Symbols.find_as(Name);
+
+ // FIXME: Can we eliminate this possibility if we support query binding?
+ if (I == JD->Symbols.end())
+ return WaitingOnGraph::ExternalState::Failed;
+
+ if (I->second.getFlags().hasError())
+ return WaitingOnGraph::ExternalState::Failed;
+
+ if (I->second.getState() == SymbolState::Ready)
+ return WaitingOnGraph::ExternalState::Ready;
+
+ return WaitingOnGraph::ExternalState::None;
+}
+
+template <typename UpdateSymbolFn, typename UpdateQueryFn>
+void ExecutionSession::IL_collectQueries(
+ JITDylib::AsynchronousSymbolQuerySet &Qs,
+ WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
+ UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
+
+ for (auto &[JD, Symbols] : QualifiedSymbols) {
+ // IL_emit and JITDylib removal are synchronized by the session lock.
+ // Since JITDylib removal removes any contained nodes from the
+ // WaitingOnGraph, we should be able to assert that all nodes in the
+ // WaitingOnGraph have not been removed.
+ assert(JD->State == JITDylib::Open &&
+ "WaitingOnGraph includes definition in defunct JITDylib");
+ for (auto &Symbol : Symbols) {
+ // Update symbol table.
+ auto I = JD->Symbols.find_as(Symbol);
+ assert(I != JD->Symbols.end() &&
+ "Failed Symbol missing from JD symbol table");
+ auto &Entry = I->second;
+ UpdateSymbol(Entry);
+
+ // Collect queries.
+ auto J = JD->MaterializingInfos.find_as(Symbol);
+ if (J != JD->MaterializingInfos.end()) {
+ for (auto &Q : J->second.takeAllPendingQueries()) {
+ UpdateQuery(*Q, *JD, Symbol, Entry);
+ Qs.insert(std::move(Q));
}
+ JD->MaterializingInfos.erase(J);
}
-
- if (!NewDepsForJD.empty())
- EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
}
}
-
- // 4. Build the worklist.
- std::deque<JITDylib::EmissionDepUnit *> Worklist;
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- // If this EDU has extra-finalization dependencies and intra-finalization
- // users then add it to the worklist.
- if (!EDU->Dependencies.empty()) {
- auto I = EDUInfos.find(EDU);
- if (I != EDUInfos.end()) {
- auto &EDUInfo = I->second;
- if (!EDUInfo.IntraEmitUsers.empty()) {
- EDUInfo.NewDeps = EDU->Dependencies;
- Worklist.push_back(EDU);
- }
- }
- }
- }
-
- // 4. Propagate dependencies through the EDU graph.
- propagateExtraEmitDeps(
- Worklist, EDUInfos,
- [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
-
- return EDUInfos;
}
-void ExecutionSession::IL_makeEDUReady(
- std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries) {
-
- // The symbols for this EDU are ready.
- auto &JD = *EDU->JD;
-
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
- "JD does not have an entry for Sym");
- auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
-
- assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
- Entry.getState() == SymbolState::Materializing) ||
- Entry.getState() == SymbolState::Resolved ||
- Entry.getState() == SymbolState::Emitted) &&
- "Emitting from state other than Resolved");
-
- Entry.setState(SymbolState::Ready);
-
- auto MII = JD.MaterializingInfos.find(SymbolStringPtr(Sym));
-
- // Check for pending queries.
- if (MII == JD.MaterializingInfos.end())
- continue;
- auto &MI = MII->second;
-
- for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) {
- Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
- if (Q->isComplete())
- Queries.insert(Q);
- Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
- }
-
- JD.MaterializingInfos.erase(MII);
- }
-
- JD.shrinkMaterializationInfoMemory();
-}
-
-void ExecutionSession::IL_makeEDUEmitted(
- std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries) {
-
- // The symbols for this EDU are emitted, but not ready.
- auto &JD = *EDU->JD;
-
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
- "JD does not have an entry for Sym");
- auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
-
- assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
- Entry.getState() == SymbolState::Materializing) ||
- Entry.getState() == SymbolState::Resolved ||
- Entry.getState() == SymbolState::Emitted) &&
- "Emitting from state other than Resolved");
-
- if (Entry.getState() == SymbolState::Emitted) {
- // This was already emitted, so we can skip the rest of this loop.
-#ifndef NDEBUG
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
- "JD does not have an entry for Sym");
- auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
- assert(Entry.getState() == SymbolState::Emitted &&
- "Symbols for EDU in inconsistent state");
- assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
- "Emitted symbol has no MI");
- auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
- assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
- "Already-emitted symbol has waiting-on-emitted queries");
- }
-#endif // NDEBUG
- break;
- }
-
- Entry.setState(SymbolState::Emitted);
- auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
- MI.DefiningEDU = EDU;
-
- for (auto &Q : MI.takeQueriesMeeting(SymbolState::Emitted)) {
- Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
- if (Q->isComplete())
- Queries.insert(Q);
- }
- }
-
- for (auto &[DepJD, Deps] : EDU->Dependencies) {
- for (auto &Dep : Deps)
- DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
- EDU.get());
- }
-}
-
-/// Removes the given dependence from EDU. If EDU's dependence set becomes
-/// empty then this function adds an entry for it to the EDUInfos map.
-/// Returns true if a new EDUInfosMap entry is added.
-bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
- JITDylib &DepJD,
- NonOwningSymbolStringPtr DepSym,
- EDUInfosMap &EDUInfos) {
- assert(EDU.Dependencies.count(&DepJD) &&
- "JD does not appear in Dependencies of DependantEDU");
- assert(EDU.Dependencies[&DepJD].count(DepSym) &&
- "Symbol does not appear in Dependencies of DependantEDU");
- auto &JDDeps = EDU.Dependencies[&DepJD];
- JDDeps.erase(DepSym);
- if (JDDeps.empty()) {
- EDU.Dependencies.erase(&DepJD);
- if (EDU.Dependencies.empty()) {
- // If the dependencies set has become empty then EDU _may_ be ready
- // (we won't know for sure until we've propagated the extra-emit deps).
- // Create an EDUInfo for it (if it doesn't have one already) so that
- // it'll be visited after propagation.
- auto &DepEDUInfo = EDUInfos[&EDU];
- if (!DepEDUInfo.EDU) {
- assert(EDU.JD->Symbols.count(
- SymbolStringPtr(EDU.Symbols.begin()->first)) &&
- "Missing symbol entry for first symbol in EDU");
- auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
- SymbolStringPtr(EDU.Symbols.begin()->first));
- assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
- "Missing MI for first symbol in DependantEDU");
- DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
- return true;
- }
- }
- }
- return false;
-}
-
-Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &ClosedJD) {
- SymbolNameSet FailedSymbols;
- for (auto &[Sym, Flags] : EDU.Symbols)
- FailedSymbols.insert(SymbolStringPtr(Sym));
- SymbolDependenceMap BadDeps;
- for (auto &Dep : EDU.Dependencies[&ClosedJD])
- BadDeps[&ClosedJD].insert(SymbolStringPtr(Dep));
- return make_error<UnsatisfiedSymbolDependencies>(
- ClosedJD.getExecutionSession().getSymbolStringPool(), EDU.JD,
- std::move(FailedSymbols), std::move(BadDeps),
- ClosedJD.getName() + " is closed");
-}
-
-Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &BadJD,
- SymbolNameSet BadDeps) {
- SymbolNameSet FailedSymbols;
- for (auto &[Sym, Flags] : EDU.Symbols)
- FailedSymbols.insert(SymbolStringPtr(Sym));
- SymbolDependenceMap BadDepsMap;
- BadDepsMap[&BadJD] = std::move(BadDeps);
- return make_error<UnsatisfiedSymbolDependencies>(
- BadJD.getExecutionSession().getSymbolStringPool(), &BadJD,
- std::move(FailedSymbols), std::move(BadDepsMap),
- "dependencies removed or in error state");
-}
-
-Expected<JITDylib::AsynchronousSymbolQuerySet>
+Expected<ExecutionSession::EmitQueries>
ExecutionSession::IL_emit(MaterializationResponsibility &MR,
- EDUInfosMap EDUInfos) {
+ WaitingOnGraph::SimplifyResult SR) {
if (MR.RT->isDefunct())
return make_error<ResourceTrackerDefunct>(MR.RT);
@@ -3279,169 +2952,50 @@ ExecutionSession::IL_emit(MaterializationResponsibility &MR,
return make_error<StringError>("JITDylib " + TargetJD.getName() +
" is defunct",
inconvertibleErrorCode());
+
#ifdef EXPENSIVE_CHECKS
verifySessionState("entering ExecutionSession::IL_emit");
#endif
- // Walk all EDUs:
- // 1. Verifying that dependencies are available (not removed or in the error
- // state.
- // 2. Removing any dependencies that are already Ready.
- // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
- // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
- std::deque<JITDylib::EmissionDepUnit *> Worklist;
- for (auto &[EDU, _] : EDUInfos)
- Worklist.push_back(EDU);
-
- for (auto *EDU : Worklist) {
- auto *EDUInfo = &EDUInfos[EDU];
-
- SmallVector<JITDylib *> DepJDsToRemove;
- for (auto &[DepJD, Deps] : EDU->Dependencies) {
- if (DepJD->State != JITDylib::Open)
- return makeJDClosedError(*EDU, *DepJD);
-
- SymbolNameSet BadDeps;
- SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
- for (auto &Dep : Deps) {
- auto DepEntryItr = DepJD->Symbols.find(SymbolStringPtr(Dep));
-
- // If this dep has been removed or moved to the error state then add it
- // to the bad deps set. We aggregate these bad deps for more
- // comprehensive error messages.
- if (DepEntryItr == DepJD->Symbols.end() ||
- DepEntryItr->second.getFlags().hasError()) {
- BadDeps.insert(SymbolStringPtr(Dep));
- continue;
- }
-
- // If this dep isn't emitted yet then just add it to the NewDeps set to
- // be propagated.
- auto &DepEntry = DepEntryItr->second;
- if (DepEntry.getState() < SymbolState::Emitted) {
- EDUInfo->NewDeps[DepJD].insert(Dep);
- continue;
- }
-
- // This dep has been emitted, so add it to the list to be removed from
- // EDU.
- DepsToRemove.push_back(Dep);
-
- // If Dep is Ready then there's nothing further to do.
- if (DepEntry.getState() == SymbolState::Ready) {
- assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
- "Unexpected MaterializationInfo attached to ready symbol");
- continue;
- }
+ auto ER = G.emit(std::move(SR),
+ [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
+ return IL_getSymbolState(JD, Name);
+ });
- // If we get here then Dep is Emitted. We need to look up its defining
- // EDU and add this EDU to the defining EDU's list of users (this means
- // creating an EDUInfos entry if the defining EDU doesn't have one
- // already).
- assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
- "Expected MaterializationInfo for emitted dependency");
- auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
- assert(DepMI.DefiningEDU &&
- "Emitted symbol does not have a defining EDU");
- assert(DepMI.DependantEDUs.empty() &&
- "Already-emitted symbol has dependant EDUs?");
- auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
- if (!DepEDUInfo.EDU) {
- // No EDUInfo yet -- build initial entry, and reset the EDUInfo
- // pointer, which we will have invalidated.
- EDUInfo = &EDUInfos[EDU];
- DepEDUInfo.EDU = DepMI.DefiningEDU;
- for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
- if (DepDepJD == &TargetJD) {
- for (auto &DepDep : DepDeps)
- if (!MR.getSymbols().count(SymbolStringPtr(DepDep)))
- DepEDUInfo.NewDeps[DepDepJD].insert(DepDep);
- } else
- DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
- }
- }
- DepEDUInfo.IntraEmitUsers.insert(EDU);
- }
+ EmitQueries EQ;
- // Some dependencies were removed or in an error state -- error out.
- if (!BadDeps.empty())
- return makeUnsatisfiedDepsError(*EDU, *DepJD, std::move(BadDeps));
-
- // Remove the emitted / ready deps from DepJD.
- for (auto &Dep : DepsToRemove)
- Deps.erase(Dep);
-
- // If there are no further deps in DepJD then flag it for removal too.
- if (Deps.empty())
- DepJDsToRemove.push_back(DepJD);
- }
-
- // Remove any JDs whose dependence sets have become empty.
- for (auto &DepJD : DepJDsToRemove) {
- assert(EDU->Dependencies.count(DepJD) &&
- "Trying to remove non-existent dep entries");
- EDU->Dependencies.erase(DepJD);
- }
-
- // Now look for users of this EDU.
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
- "Sym not present in symbol table");
- assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
- SymbolState::Resolved ||
- TargetJD.Symbols[SymbolStringPtr(Sym)]
- .getFlags()
- .hasMaterializationSideEffectsOnly()) &&
- "Emitting symbol not in the resolved state");
- assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
- "Symbol is already in an error state");
-
- auto MII = TargetJD.MaterializingInfos.find(SymbolStringPtr(Sym));
- if (MII == TargetJD.MaterializingInfos.end() ||
- MII->second.DependantEDUs.empty())
- continue;
-
- for (auto &DependantEDU : MII->second.DependantEDUs) {
- if (IL_removeEDUDependence(*DependantEDU, TargetJD, Sym, EDUInfos))
- EDUInfo = &EDUInfos[EDU];
- EDUInfo->IntraEmitUsers.insert(DependantEDU);
- }
- MII->second.DependantEDUs.clear();
- }
- }
-
- Worklist.clear();
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
- if (EDUInfo.NewDeps.empty())
- EDUInfo.NewDeps = EDU->Dependencies;
- Worklist.push_back(EDU);
- }
- }
-
- propagateExtraEmitDeps(
- Worklist, EDUInfos,
- [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
- NonOwningSymbolStringPtr Sym) {
- JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(&EDU);
- });
+ // Handle failed queries.
+ for (auto &SN : ER.Failed)
+ IL_collectQueries(
+ EQ.Failed, SN->defs(),
+ [](JITDylib::SymbolTableEntry &E) {
+ E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
+ },
+ [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
+ NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
+ auto &FS = EQ.FailedSymsForQuery[&Q];
+ if (!FS)
+ FS = std::make_shared<SymbolDependenceMap>();
+ (*FS)[&JD].insert(SymbolStringPtr(Name));
+ });
- JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
+ for (auto &FQ : EQ.Failed)
+ FQ->detach();
- // Extract completed queries and lodge not-yet-ready EDUs in the
- // session.
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- if (EDU->Dependencies.empty())
- IL_makeEDUReady(std::move(EDUInfo.EDU), CompletedQueries);
- else
- IL_makeEDUEmitted(std::move(EDUInfo.EDU), CompletedQueries);
- }
+ for (auto &SN : ER.Ready)
+ IL_collectQueries(
+ EQ.Updated, SN->defs(),
+ [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
+ [](AsynchronousSymbolQuery &Q, JITDylib &JD,
+ NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
+ Q.notifySymbolMetRequiredState(SymbolStringPtr(Name), E.getSymbol());
+ });
#ifdef EXPENSIVE_CHECKS
verifySessionState("exiting ExecutionSession::IL_emit");
#endif
- return std::move(CompletedQueries);
+ return std::move(EQ);
}
Error ExecutionSession::OL_notifyEmitted(
@@ -3471,40 +3025,107 @@ Error ExecutionSession::OL_notifyEmitted(
}
#endif // NDEBUG
- auto EDUInfos = simplifyDepGroups(MR, DepGroups);
+ std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
+ WaitingOnGraph::ContainerElementsMap Residual;
+ {
+ auto &JDResidual = Residual[&MR.getTargetJITDylib()];
+ for (auto &[Name, Flags] : MR.getSymbols())
+ JDResidual.insert(NonOwningSymbolStringPtr(Name));
+
+ for (auto &SDG : DepGroups) {
+ WaitingOnGraph::ContainerElementsMap Defs;
+ assert(!SDG.Symbols.empty());
+ auto &JDDefs = Defs[&MR.getTargetJITDylib()];
+ for (auto &Def : SDG.Symbols) {
+ JDDefs.insert(NonOwningSymbolStringPtr(Def));
+ JDResidual.erase(NonOwningSymbolStringPtr(Def));
+ }
+ WaitingOnGraph::ContainerElementsMap Deps;
+ if (!SDG.Dependencies.empty()) {
+ for (auto &[JD, Syms] : SDG.Dependencies) {
+ auto &JDDeps = Deps[JD];
+ for (auto &Dep : Syms)
+ JDDeps.insert(NonOwningSymbolStringPtr(Dep));
+ }
+ }
+ SNs.push_back(std::make_unique<WaitingOnGraph::SuperNode>(
+ std::move(Defs), std::move(Deps)));
+ }
+ if (!JDResidual.empty())
+ SNs.push_back(std::make_unique<WaitingOnGraph::SuperNode>(
+ std::move(Residual), WaitingOnGraph::ContainerElementsMap()));
+ }
+
+ auto SR = WaitingOnGraph::simplify(std::move(SNs));
LLVM_DEBUG({
dbgs() << " Simplified dependencies:\n";
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- dbgs() << " Symbols: { ";
- for (auto &[Sym, Flags] : EDU->Symbols)
- dbgs() << Sym << " ";
- dbgs() << "}, Dependencies: { ";
- for (auto &[DepJD, Deps] : EDU->Dependencies) {
- dbgs() << "(" << DepJD->getName() << ", { ";
- for (auto &Dep : Deps)
- dbgs() << Dep << " ";
- dbgs() << "}) ";
+ for (auto &SN : SR.superNodes()) {
+ dbgs() << " Defs: {";
+ for (auto &[JD, Syms] : SN->defs()) {
+ dbgs() << " (" << JD->getName() << ", [";
+ for (auto &Sym : Syms)
+ dbgs() << " " << Sym;
+ dbgs() << " ])";
+ }
+ dbgs() << " }, Deps: {";
+ for (auto &[JD, Syms] : SN->deps()) {
+ dbgs() << " (" << JD->getName() << ", [";
+ for (auto &Sym : Syms)
+ dbgs() << " " << Sym;
+ dbgs() << " ])";
}
- dbgs() << "}\n";
+ dbgs() << " }\n";
}
});
-
- auto CompletedQueries =
- runSessionLocked([&]() { return IL_emit(MR, EDUInfos); });
+ auto EmitQueries =
+ runSessionLocked([&]() { return IL_emit(MR, std::move(SR)); });
// On error bail out.
- if (!CompletedQueries)
- return CompletedQueries.takeError();
+ if (!EmitQueries)
+ return EmitQueries.takeError();
- MR.SymbolFlags.clear();
+ // Otherwise notify failed queries, and any updated queries that have been
+ // completed.
- // Otherwise notify all the completed queries.
- for (auto &Q : *CompletedQueries) {
- assert(Q->isComplete() && "Q is not complete");
- Q->handleComplete(*this);
+ // FIXME: Get rid of error return from notifyEmitted.
+ SymbolDependenceMap BadDeps;
+ {
+ for (auto &FQ : EmitQueries->Failed) {
+ FQ->detach();
+ assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
+ "Missing failed symbols for query");
+ auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
+ for (auto &[JD, Syms] : *FailedSyms) {
+ auto &BadDepsForJD = BadDeps[JD];
+ for (auto &Sym : Syms)
+ BadDepsForJD.insert(Sym);
+ }
+ FQ->handleFailed(make_error<FailedToMaterialize>(getSymbolStringPool(),
+ std::move(FailedSyms)));
+ }
+ }
+
+ for (auto &UQ : EmitQueries->Updated)
+ if (UQ->isComplete())
+ UQ->handleComplete(*this);
+
+ // If there are any bad dependencies then return an error.
+ if (!BadDeps.empty()) {
+ SymbolNameSet BadNames;
+ // Note: The name set calculated here is bogus: it includes all symbols in
+ // the MR, not just the ones that failed. We want to remove the error
+ // return path from notifyEmitted anyway, so this is just a brief
+ // placeholder to maintain (roughly) the current error behavior.
+ for (auto &[Name, Flags] : MR.getSymbols())
+ BadNames.insert(Name);
+ MR.SymbolFlags.clear();
+ return make_error<UnsatisfiedSymbolDependencies>(
+ getSymbolStringPool(), &MR.getTargetJITDylib(), std::move(BadNames),
+ std::move(BadDeps), "dependencies removed or in error state");
}
+ MR.SymbolFlags.clear();
return Error::success();
}
@@ -3535,158 +3156,48 @@ ExecutionSession::IL_failSymbols(JITDylib &JD,
#endif
JITDylib::AsynchronousSymbolQuerySet FailedQueries;
- auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
- auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
- JITDylib::AsynchronousSymbolQueryList ToDetach;
- for (auto &Q : MI.pendingQueries()) {
- // Add the query to the list to be failed and detach it.
- FailedQueries.insert(Q);
- ToDetach.push_back(Q);
+ auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
+ auto I = FailJD->Symbols.find_as(FailSym);
+ assert(I != FailJD->Symbols.end());
+ I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
+ auto J = FailJD->MaterializingInfos.find_as(FailSym);
+ if (J != FailJD->MaterializingInfos.end()) {
+ for (auto &Q : J->second.takeAllPendingQueries())
+ FailedQueries.insert(std::move(Q));
+ FailJD->MaterializingInfos.erase(J);
}
- for (auto &Q : ToDetach)
- Q->detach();
- assert(!MI.hasQueriesPending() && "Queries still pending after detach");
};
- for (auto &Name : SymbolsToFail) {
- (*FailedSymbolsMap)[&JD].insert(Name);
-
- // Look up the symbol to fail.
- auto SymI = JD.Symbols.find(Name);
-
- // FIXME: Revisit this. We should be able to assert sequencing between
- // ResourceTracker removal and symbol failure.
- //
- // It's possible that this symbol has already been removed, e.g. if a
- // materialization failure happens concurrently with a ResourceTracker or
- // JITDylib removal. In that case we can safely skip this symbol and
- // continue.
- if (SymI == JD.Symbols.end())
- continue;
- auto &Sym = SymI->second;
-
- // If the symbol is already in the error state then we must have visited
- // it earlier.
- if (Sym.getFlags().hasError()) {
- assert(!JD.MaterializingInfos.count(Name) &&
- "Symbol in error state still has MaterializingInfo");
- continue;
- }
+ auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
- // Move the symbol into the error state.
- Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
-
- // FIXME: Come up with a sane mapping of state to
- // presence-of-MaterializingInfo so that we can assert presence / absence
- // here, rather than testing it.
- auto MII = JD.MaterializingInfos.find(Name);
- if (MII == JD.MaterializingInfos.end())
- continue;
-
- auto &MI = MII->second;
-
- // Collect queries to be failed for this MII.
- ExtractFailedQueries(MI);
-
- if (MI.DefiningEDU) {
- // If there is a DefiningEDU for this symbol then remove this
- // symbol from it.
- assert(MI.DependantEDUs.empty() &&
- "Symbol with DefiningEDU should not have DependantEDUs");
- assert(Sym.getState() >= SymbolState::Emitted &&
- "Symbol has EDU, should have been emitted");
- assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
- "Symbol does not appear in its DefiningEDU");
- MI.DefiningEDU->Symbols.erase(NonOwningSymbolStringPtr(Name));
-
- // Remove this EDU from the dependants lists of its dependencies.
- for (auto &[DepJD, DepSyms] : MI.DefiningEDU->Dependencies) {
- for (auto DepSym : DepSyms) {
- assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
- "DepSym not in DepJD");
- assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
- "DepSym has not MaterializingInfo");
- auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
- assert(SymMI.DependantEDUs.count(MI.DefiningEDU.get()) &&
- "DefiningEDU missing from DependantEDUs list of dependency");
- SymMI.DependantEDUs.erase(MI.DefiningEDU.get());
- }
- }
+ {
+ auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
+ for (auto &Sym : SymbolsToFail) {
+ FailedSymsForJD.insert(Sym);
+ Fail(&JD, NonOwningSymbolStringPtr(Sym));
+ }
+ }
- MI.DefiningEDU = nullptr;
- } else {
- // Otherwise if there are any EDUs waiting on this symbol then move
- // those symbols to the error state too, and deregister them from the
- // symbols that they depend on.
- // Note: We use a copy of DependantEDUs here since we'll be removing
- // from the original set as we go.
- for (auto &DependantEDU : MI.DependantEDUs) {
-
- // Remove DependantEDU from all of its users DependantEDUs lists.
- for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
- for (auto DepSym : DepSyms) {
- // Skip self-reference to avoid invalidating the MI.DependantEDUs
- // map. We'll clear this later.
- if (DepJD == &JD && DepSym == Name)
- continue;
- assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
- "DepSym not in DepJD?");
- assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
- "DependantEDU not registered with symbol it depends on");
- auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
- assert(SymMI.DependantEDUs.count(DependantEDU) &&
- "DependantEDU missing from DependantEDUs list");
- SymMI.DependantEDUs.erase(DependantEDU);
- }
- }
+ WaitingOnGraph::ContainerElementsMap ToFail;
+ auto &JDToFail = ToFail[&JD];
+ for (auto &Sym : SymbolsToFail)
+ JDToFail.insert(NonOwningSymbolStringPtr(Sym));
- // Move any symbols defined by DependantEDU into the error state and
- // fail any queries waiting on them.
- auto &DepJD = *DependantEDU->JD;
- auto DepEDUSymbols = std::move(DependantEDU->Symbols);
- for (auto &[DepName, Flags] : DepEDUSymbols) {
- auto DepSymItr = DepJD.Symbols.find(SymbolStringPtr(DepName));
- assert(DepSymItr != DepJD.Symbols.end() &&
- "Symbol not present in table");
- auto &DepSym = DepSymItr->second;
-
- assert(DepSym.getState() >= SymbolState::Emitted &&
- "Symbol has EDU, should have been emitted");
- assert(!DepSym.getFlags().hasError() &&
- "Symbol is already in the error state?");
- DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
- (*FailedSymbolsMap)[&DepJD].insert(SymbolStringPtr(DepName));
-
- // This symbol has a defining EDU so its MaterializingInfo object must
- // exist.
- auto DepMIItr =
- DepJD.MaterializingInfos.find(SymbolStringPtr(DepName));
- assert(DepMIItr != DepJD.MaterializingInfos.end() &&
- "Symbol has defining EDU but not MaterializingInfo");
- auto &DepMI = DepMIItr->second;
- assert(DepMI.DefiningEDU.get() == DependantEDU &&
- "Bad EDU dependence edge");
- assert(DepMI.DependantEDUs.empty() &&
- "Symbol was emitted, should not have any DependantEDUs");
- ExtractFailedQueries(DepMI);
- DepJD.MaterializingInfos.erase(SymbolStringPtr(DepName));
- }
+ auto FailedSNs = G.fail(ToFail);
- DepJD.shrinkMaterializationInfoMemory();
+ for (auto &SN : FailedSNs) {
+ for (auto &[FailJD, Defs] : SN->defs()) {
+ auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
+ for (auto &Def : Defs) {
+ FailedSymsForFailJD.insert(SymbolStringPtr(Def));
+ Fail(FailJD, Def);
}
-
- MI.DependantEDUs.clear();
}
-
- assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
- assert(MI.DependantEDUs.empty() &&
- "DependantEDUs should have been removed above");
- assert(!MI.hasQueriesPending() &&
- "Can not delete MaterializingInfo with queries pending");
- JD.MaterializingInfos.erase(Name);
}
- JD.shrinkMaterializationInfoMemory();
+ // Detach all failed queries.
+ for (auto &Q : FailedQueries)
+ Q->detach();
#ifdef EXPENSIVE_CHECKS
verifySessionState("exiting ExecutionSession::IL_failSymbols");
@@ -3721,9 +3232,11 @@ void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
return IL_failSymbols(MR.getTargetJITDylib(), SymbolsToFail);
});
- for (auto &Q : FailedQueries)
+ for (auto &Q : FailedQueries) {
+ Q->detach();
Q->handleFailed(
make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
+ }
}
Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
index 87d757805a64c..03a06792b4a30 100644
--- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
@@ -448,7 +448,7 @@ Error SimpleRemoteEPC::handleHangup(SimpleRemoteEPCArgBytesVector ArgBytes) {
if (const char *ErrMsg = WFR.getOutOfBandError())
return make_error<StringError>(ErrMsg, inconvertibleErrorCode());
- detail::SPSSerializableError Info;
+ orc::shared::detail::SPSSerializableError Info;
SPSInputBuffer IB(WFR.data(), WFR.size());
if (!SPSArgList<SPSError>::deserialize(IB, Info))
return make_error<StringError>("Could not deserialize hangup info",
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s
index 454079923622e..e52e4367f97f7 100644
--- a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s
@@ -1,5 +1,5 @@
# RUN: llvm-mc -triple=arm64e-apple-macosx -filetype=obj -o %t.o %s
-# RUN: llvm-jitlink -num-threads=0 -debug-only=orc -noexec \
+# RUN: llvm-jitlink -debug-only=orc -noexec \
# RUN: -abs _foo=0x1 %t.o 2>&1 \
# RUN: | FileCheck %s
#
diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s b/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s
index 83d71cdf6fc83..529395822f5f7 100644
--- a/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s
+++ b/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s
@@ -16,8 +16,7 @@
# CHECK-DAG: Symbols: { _foo }, Dependencies: { (main, { _external_func }) }
# CHECK-DAG: Symbols: { _baz }, Dependencies: { (main, { _foo }) }
# CHECK: Simplified dependencies:
-# CHECK-DAG: Symbols: { _foo }, Dependencies: { (main, { _external_func }) }
-# CHECK-DAG: Symbols: { _baz }, Dependencies: { (main, { _external_func }) }
+# CHECK-DAG: Defs: { (main, [ _baz _foo ]) }, Deps: { (main, [ _external_func ]) }
.section __TEXT,__text,regular,pure_instructions
diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
index a2bbb10039c9a..7e3ebc88cea63 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
@@ -42,6 +42,7 @@ add_llvm_unittest(OrcJITTests
SymbolStringPoolTest.cpp
TaskDispatchTest.cpp
ThreadSafeModuleTest.cpp
+ WaitingOnGraphTest.cpp
WrapperFunctionUtilsTest.cpp
JITLinkRedirectionManagerTest.cpp
ReOptimizeLayerTest.cpp
diff --git a/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp
new file mode 100644
index 0000000000000..b988a78a3783a
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp
@@ -0,0 +1,553 @@
+//===--------- WaitingOnGraphTest.cpp - Test WaitingOnGraph APIs ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/WaitingOnGraph.h"
+#include "gtest/gtest.h"
+
+namespace llvm::orc::detail {
+
+class WaitingOnGraphTest : public testing::Test {
+public:
+ using TestGraph = WaitingOnGraph<uintptr_t, uintptr_t>;
+
+protected:
+ using SuperNode = TestGraph::SuperNode;
+ using SuperNodeBuilder = TestGraph::SuperNodeBuilder;
+ using ContainerElementsMap = TestGraph::ContainerElementsMap;
+ using ElemToSuperNodeMap = TestGraph::ElemToSuperNodeMap;
+ using SimplifyResult = TestGraph::SimplifyResult;
+ using EmitResult = TestGraph::EmitResult;
+
+ static const ContainerElementsMap &getDefs(SuperNode &SN) { return SN.Defs; }
+
+ static const ContainerElementsMap &getDeps(SuperNode &SN) { return SN.Deps; }
+
+ static std::vector<std::unique_ptr<SuperNode>> &getSNs(SimplifyResult &SR) {
+ return SR.SNs;
+ }
+
+ static ElemToSuperNodeMap &getElemToSN(SimplifyResult &SR) {
+ return SR.ElemToSN;
+ }
+
+ static std::vector<std::unique_ptr<SuperNode>> &getPendingSNs(TestGraph &G) {
+ return G.PendingSNs;
+ }
+
+ static ContainerElementsMap merge(ContainerElementsMap M1,
+ const ContainerElementsMap &M2) {
+ ContainerElementsMap Result = std::move(M1);
+ for (auto &[Container, Elems] : M2)
+ Result[Container].insert(Elems.begin(), Elems.end());
+ return Result;
+ }
+
+ ContainerElementsMap
+ collapseDefs(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ bool DepsMustMatch = true) {
+ if (SNs.empty())
+ return ContainerElementsMap();
+
+ ContainerElementsMap Result = SNs[0]->defs();
+ const ContainerElementsMap &Deps = SNs[0]->deps();
+
+ for (size_t I = 1; I != SNs.size(); ++I) {
+ assert(!DepsMustMatch || SNs[I]->deps() == Deps);
+ Result = merge(std::move(Result), SNs[I]->defs());
+ }
+
+ return Result;
+ }
+
+ EmitResult integrate(EmitResult ER) {
+ for (auto &SN : ER.Ready)
+ for (auto &[Container, Elems] : SN->defs())
+ Ready[Container].insert(Elems.begin(), Elems.end());
+ for (auto &SN : ER.Failed)
+ for (auto &[Container, Elems] : SN->defs())
+ Failed[Container].insert(Elems.begin(), Elems.end());
+ return ER;
+ }
+
+ EmitResult emit(SimplifyResult SR) {
+ return integrate(G.emit(std::move(SR), GetExternalState));
+ }
+
+ TestGraph G;
+ ContainerElementsMap Ready;
+ ContainerElementsMap Failed;
+
+ class ExternalStateGetter {
+ public:
+ ExternalStateGetter(WaitingOnGraphTest &T) : T(T) {}
+ TestGraph::ExternalState operator()(TestGraph::ContainerId C,
+ TestGraph::ElementId E) {
+ {
+ auto I = T.Failed.find(C);
+ if (I != T.Failed.end())
+ if (I->second.count(E))
+ return TestGraph::ExternalState::Failed;
+ }
+
+ {
+ auto I = T.Ready.find(C);
+ if (I != T.Ready.end())
+ if (I->second.count(E))
+ return TestGraph::ExternalState::Ready;
+ }
+
+ return TestGraph::ExternalState::None;
+ }
+
+ private:
+ WaitingOnGraphTest &T;
+ };
+
+ ExternalStateGetter GetExternalState{*this};
+};
+
+} // namespace llvm::orc::detail
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::orc::detail;
+
+TEST_F(WaitingOnGraphTest, ConstructAndDestroyEmpty) {
+ // Nothing to do here -- we're just testing construction and destruction
+ // of the WaitingOnGraphTest::G member.
+}
+
+TEST_F(WaitingOnGraphTest, Build_TrivialSingleSuperNode) {
+ // Add one set of trivial defs and empty deps to the builder, make sure that
+ // they're passed through to the resulting super-node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0}}});
+ ContainerElementsMap Deps;
+ B.add(Defs, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), Defs);
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_EmptyDefs) {
+ // Adding empty def sets is ok, but should not result in creation of a
+ // SuperNode.
+ SuperNodeBuilder B;
+ ContainerElementsMap Empty;
+ B.add(Empty, Empty);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_TRUE(SNs.empty());
+}
+
+TEST_F(WaitingOnGraphTest, Build_NonTrivialSingleSuperNode) {
+ // Add one non-trivwial set of defs and deps. Make sure that they're passed
+ // through to the resulting super-node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0, 1, 2}}});
+ ContainerElementsMap Deps({{1, {3, 4, 5}}});
+ B.add(Defs, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), Defs);
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_CoalesceEmptyDeps) {
+ // Add two trivial defs both with empty deps to the builder. Check that
+ // they're coalesced into a single super-node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs1({{0, {0}}});
+ ContainerElementsMap Defs2({{0, {1}}});
+ ContainerElementsMap Deps;
+ B.add(Defs1, Deps);
+ B.add(Defs2, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), merge(Defs1, Defs2));
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_CoalesceNonEmptyDeps) {
+ // Add two sets trivial of trivial defs with empty deps to the builder. Check
+ // that the two coalesce into a single super node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs1({{0, {0}}});
+ ContainerElementsMap Defs2({{0, {1}}});
+ ContainerElementsMap Deps({{1, {1}}});
+ B.add(Defs1, Deps);
+ B.add(Defs2, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), merge(Defs1, Defs2));
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_CoalesceInterleaved) {
+ // Add multiple sets of defs, some with the same dep sets. Check that nodes
+ // are still coalesced as expected.
+ SuperNodeBuilder B;
+
+ ContainerElementsMap DefsA1({{0, {0}}});
+ ContainerElementsMap DefsA2({{0, {1}}});
+ ContainerElementsMap DefsB1({{1, {0}}});
+ ContainerElementsMap DefsB2({{1, {1}}});
+ ContainerElementsMap DepsA({{2, {0}}, {3, {0}}});
+ ContainerElementsMap DepsB({{4, {0}}, {5, {0}}});
+ B.add(DefsA1, DepsA);
+ B.add(DefsB1, DepsB);
+ B.add(DefsA2, DepsA);
+ B.add(DefsB2, DepsB);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 2U);
+ EXPECT_EQ(getDefs(*SNs[0]), merge(DefsA1, DefsA2));
+ EXPECT_EQ(getDeps(*SNs[0]), DepsA);
+ EXPECT_EQ(getDefs(*SNs[1]), merge(DefsB1, DefsB2));
+ EXPECT_EQ(getDeps(*SNs[1]), DepsB);
+}
+
+TEST_F(WaitingOnGraphTest, Build_SelfDepRemoval) {
+ // Add multiple sets of defs, some with the same dep sets. Check that nodes
+ // are still coalesced as expected.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0, 1}}});
+ ContainerElementsMap Deps({{0, {1}}});
+ ContainerElementsMap Empty;
+ B.add(Defs, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), Defs);
+ EXPECT_EQ(getDeps(*SNs[0]), Empty);
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_EmptySimplification) {
+ auto SR = TestGraph::simplify({});
+ auto &SNs = getSNs(SR);
+ EXPECT_EQ(SNs.size(), 0U);
+ EXPECT_EQ(getElemToSN(SR), ElemToSuperNodeMap());
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_TrivialSingleSuperNode) {
+ // Test trivial call to simplify.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0}}});
+ ContainerElementsMap Deps({{0, {0}}});
+ B.add(Defs, Deps);
+ auto SR = TestGraph::simplify(B.takeSuperNodes());
+ ContainerElementsMap Empty;
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs.at(0)), Defs);
+ EXPECT_EQ(getDeps(*SNs.at(0)), Empty);
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ ExpectedElemToSNs[0][0] = SNs[0].get();
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_SimplifySingleContainerSimpleCycle) {
+ // Test trivial simplification call with two nodes and one internal
+ // dependence cycle within a single container:
+ // N0: (0, 0) -> (0, 1)
+ // N1: (0, 1) -> (0, 0)
+ // We expect intra-simplify cycle elimination to clear both dependence sets,
+ // and coalescing to join them into one supernode covering both defs.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {0}}});
+ B.add(Defs1, Deps1);
+ auto SR = TestGraph::simplify(B.takeSuperNodes());
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ ContainerElementsMap Empty;
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs.at(0)), merge(Defs0, Defs1));
+ EXPECT_EQ(getDeps(*SNs.at(0)), Empty);
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ ExpectedElemToSNs[0][0] = SNs[0].get();
+ ExpectedElemToSNs[0][1] = SNs[0].get();
+
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest,
+ Simplification_SimplifySingleContainerNElementCycle) {
+ // Test trivial simplification call with M nodes and one internal
+ // dependence cycle within a single container:
+ // N0: (0, 0) -> (0, 1)
+ // N1: (0, 1) -> (0, 2)
+ // ...
+ // NM: (0, M) -> (0, 0)
+ // We expect intra-simplify cycle elimination to clear all dependence sets,
+ // and coalescing to join them into one supernode covering all defs.
+ SuperNodeBuilder B;
+ constexpr size_t M = 10;
+ for (size_t I = 0; I != M; ++I) {
+ ContainerElementsMap Defs({{0, {I}}});
+ ContainerElementsMap Deps({{0, {(I + 1) % M}}});
+ B.add(Defs, Deps);
+ }
+ auto InitSNs = B.takeSuperNodes();
+ EXPECT_EQ(InitSNs.size(), M);
+
+ auto SR = TestGraph::simplify(std::move(InitSNs));
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ ContainerElementsMap ExpectedDefs;
+ for (size_t I = 0; I != M; ++I)
+ ExpectedDefs[0].insert(I);
+ ContainerElementsMap Empty;
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs.at(0)), ExpectedDefs);
+ EXPECT_EQ(getDeps(*SNs.at(0)), Empty);
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ for (size_t I = 0; I != M; ++I)
+ ExpectedElemToSNs[0][I] = SNs[0].get();
+
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_SimplifyIntraSimplifyPropagateDeps) {
+ // Test trivial simplification call with two nodes and one internal
+ // dependence cycle within a single container:
+ // N0: (0, 0) -> (0, {1, 2})
+ // N1: (0, 1) -> (0, {3})
+ // We expect intra-simplify cycle elimination to replace the dependence of
+ // (0, 0) on (0, 1) with a dependence on (0, 3) instead.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1, 2}}});
+ B.add(Defs0, Deps0);
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {3}}});
+ B.add(Defs1, Deps1);
+ auto SR = TestGraph::simplify(B.takeSuperNodes());
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ EXPECT_EQ(SNs.size(), 2U);
+
+ // ContainerElemenstMap ExpectedDefs0({{0, {0}}});
+ // ContainerElemenstMap ExpectedDeps0({{0, {1, 3}}});
+ EXPECT_EQ(getDefs(*SNs.at(0)), ContainerElementsMap({{0, {0}}}));
+ EXPECT_EQ(getDeps(*SNs.at(0)), ContainerElementsMap({{0, {2, 3}}}));
+
+ EXPECT_EQ(getDefs(*SNs.at(1)), ContainerElementsMap({{0, {1}}}));
+ EXPECT_EQ(getDeps(*SNs.at(1)), ContainerElementsMap({{0, {3}}}));
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ ExpectedElemToSNs[0][0] = SNs[0].get();
+ ExpectedElemToSNs[0][1] = SNs[1].get();
+
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_EmptyEmit) {
+ // Check that empty emits work as expected.
+ auto ER = G.emit(TestGraph::simplify({}), GetExternalState);
+
+ EXPECT_EQ(ER.Ready.size(), 0U);
+ EXPECT_EQ(ER.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_TrivialSingleNode) {
+ // Check that emitting a single node behaves as expected.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0}}});
+ B.add(Defs, ContainerElementsMap());
+ auto ER = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER.Ready), Defs);
+ EXPECT_EQ(ER.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_TrivialSequence) {
+ // Perform a sequence of two emits where the second emit depends on the
+ // first. Check that nodes become ready after each emit.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Empty;
+ B.add(Defs0, Empty);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER0.Ready), Defs0);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {0}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER1.Ready), Defs1);
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_TrivialReverseSequence) {
+ // Perform a sequence of two emits where the first emit depends on the
+ // second. Check that both nodes become ready after the second emit.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Empty;
+ B.add(Defs1, Empty);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER1.Ready), merge(Defs0, Defs1));
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_Coalescing) {
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{1, {0}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{1, {0}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER1.Ready.size(), 0U);
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+
+ // Check that after emitting two nodes with the same dep set we have only one
+ // pending supernode whose defs are the union of the defs in the two emits.
+ auto &PendingSNs = getPendingSNs(G);
+ EXPECT_EQ(PendingSNs.size(), 1U);
+ EXPECT_EQ(getDefs(*PendingSNs.at(0)), merge(Defs0, Defs1));
+
+ ContainerElementsMap Defs2({{1, {0}}});
+ ContainerElementsMap Empty;
+ B.add(Defs2, Empty);
+ auto ER2 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER2.Ready), merge(merge(Defs0, Defs1), Defs2));
+ EXPECT_EQ(ER2.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_ZigZag) {
+ // Perform a sequence of four emits, where the first three contain a zig-zag
+ // pattern:
+ // 1. (0, 0) -> (0, 1)
+ // 2. (0, 2) -> (0, 3)
+ // ^ -- At this point we expect two pending supernodes.
+ // 3. (0, 1) -> (0, 2)
+ // ^ -- Resolution of (0, 1) should cause all three emitted nodes to coalsce
+ // into one supernode defining (0, {1, 2, 3}).
+ // 4. (0, 3)
+ // ^ -- Should cause all four nodes to become ready.
+
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {2}}});
+ ContainerElementsMap Deps1({{0, {3}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER1.Ready.size(), 0U);
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+
+ // Check that after emitting two nodes with the same dep set we have only one
+ // pending supernode whose defs are the union of the defs in the two emits.
+ auto &PendingSNs = getPendingSNs(G);
+ EXPECT_EQ(PendingSNs.size(), 2U);
+ EXPECT_EQ(getDefs(*PendingSNs.at(0)), Defs0);
+ EXPECT_EQ(getDeps(*PendingSNs.at(0)), Deps0);
+ EXPECT_EQ(getDefs(*PendingSNs.at(1)), Defs1);
+ EXPECT_EQ(getDeps(*PendingSNs.at(1)), Deps1);
+
+ ContainerElementsMap Defs2({{0, {1}}});
+ ContainerElementsMap Deps2({{0, {2}}});
+ B.add(Defs2, Deps2);
+ auto ER2 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER2.Ready.size(), 0U);
+ EXPECT_EQ(ER2.Failed.size(), 0U);
+
+ // Check that after emitting the third node we've coalesced all three.
+ EXPECT_EQ(PendingSNs.size(), 1U);
+ EXPECT_EQ(getDefs(*PendingSNs.at(0)), merge(merge(Defs0, Defs1), Defs2));
+ EXPECT_EQ(getDeps(*PendingSNs.at(0)), Deps1);
+
+ ContainerElementsMap Defs3({{0, {3}}});
+ ContainerElementsMap Empty;
+ B.add(Defs3, Empty);
+ auto ER3 = emit(TestGraph::simplify(B.takeSuperNodes()));
+
+ EXPECT_EQ(collapseDefs(ER3.Ready),
+ merge(merge(merge(Defs0, Defs1), Defs2), Defs3));
+ EXPECT_EQ(ER2.Failed.size(), 0U);
+ EXPECT_TRUE(PendingSNs.empty());
+}
+
+TEST_F(WaitingOnGraphTest, Fail_Empty) {
+ // Check that failing an empty set is a no-op.
+ auto FR = G.fail(ContainerElementsMap());
+ EXPECT_EQ(FR.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Fail_Single) {
+ // Check that failing a set with no existing dependencies works.
+ auto FR = G.fail({{0, {0}}});
+ EXPECT_EQ(FR.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Fail_EmitDependenceOnFailure) {
+ // Check that emitted nodes that directly depend on failed nodes also fail.
+ Failed = {{0, {0}}};
+
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {1}}});
+ ContainerElementsMap Deps({{0, {0}}});
+ B.add(Defs, Deps);
+ auto ER = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER.Ready.size(), 0U);
+ EXPECT_EQ(collapseDefs(ER.Failed, false), Defs);
+}
+
+TEST_F(WaitingOnGraphTest, Fail_ZigZag) {
+ // Check that if an emit introduces a transitive dependence of a failed
+ // node, then all nodes that depend on the failed node are also failed.
+ SuperNodeBuilder B;
+
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ Failed = {{0, {2}}};
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {2}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER1.Ready.size(), 0U);
+ EXPECT_EQ(collapseDefs(ER1.Failed, false), merge(Defs0, Defs1));
+}
More information about the llvm-commits
mailing list