[llvm] Reapply "[ORC] Replace ORC's baked-in dependence ... (#163027)" with … (PR #164340)
Lang Hames via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 20 16:46:21 PDT 2025
https://github.com/lhames created https://github.com/llvm/llvm-project/pull/164340
…fixes.
This reapplies c8c86efbbb5, which was reverted in 13ca8723d1b due to bot failures. Those failures were compilation errors exposed when LLVM is built with LLVM_ENABLE_EXPENSIVE_CHECKS=On. They have been fixed in this commit.
>From ff8e0fbe8c62a347843955d03380de81781ca56e Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames at gmail.com>
Date: Mon, 20 Oct 2025 16:24:30 -0700
Subject: [PATCH] Reapply "[ORC] Replace ORC's baked-in dependence ...
(#163027)" with fixes.
This reapplies c8c86efbbb5, which was reverted in 13ca8723d1b due to bot
failures. Those failures were compilation errors exposed when LLVM is built
with LLVM_ENABLE_EXPENSIVE_CHECKS=On. They have been fixed in this commit.
---
llvm/include/llvm/ExecutionEngine/Orc/Core.h | 66 +-
.../llvm/ExecutionEngine/Orc/WaitingOnGraph.h | 622 ++++++++++
llvm/lib/ExecutionEngine/Orc/Core.cpp | 1001 ++++-------------
.../ExecutionEngine/Orc/SimpleRemoteEPC.cpp | 2 +-
.../x86-64/LocalDependencyPropagation.s | 3 +-
.../ExecutionEngine/Orc/CMakeLists.txt | 1 +
.../Orc/WaitingOnGraphTest.cpp | 553 +++++++++
7 files changed, 1429 insertions(+), 819 deletions(-)
create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h
create mode 100644 llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index f407b56817fc3..8613ddd8e3b11 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -26,6 +26,7 @@
#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
#include "llvm/ExecutionEngine/Orc/TaskDispatch.h"
+#include "llvm/ExecutionEngine/Orc/WaitingOnGraph.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ExtensibleRTTI.h"
@@ -49,6 +50,9 @@ class InProgressLookupState;
enum class SymbolState : uint8_t;
+using WaitingOnGraph =
+ detail::WaitingOnGraph<JITDylib *, NonOwningSymbolStringPtr>;
+
using ResourceTrackerSP = IntrusiveRefCntPtr<ResourceTracker>;
using JITDylibSP = IntrusiveRefCntPtr<JITDylib>;
@@ -1131,20 +1135,6 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
using UnmaterializedInfosList =
std::vector<std::shared_ptr<UnmaterializedInfo>>;
- struct EmissionDepUnit {
- EmissionDepUnit(JITDylib &JD) : JD(&JD) {}
-
- JITDylib *JD = nullptr;
- DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> Symbols;
- DenseMap<JITDylib *, DenseSet<NonOwningSymbolStringPtr>> Dependencies;
- };
-
- struct EmissionDepUnitInfo {
- std::shared_ptr<EmissionDepUnit> EDU;
- DenseSet<EmissionDepUnit *> IntraEmitUsers;
- DenseMap<JITDylib *, DenseSet<NonOwningSymbolStringPtr>> NewDeps;
- };
-
// Information about not-yet-ready symbol.
// * DefiningEDU will point to the EmissionDepUnit that defines the symbol.
// * DependantEDUs will hold pointers to any EmissionDepUnits currently
@@ -1154,9 +1144,6 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
struct MaterializingInfo {
friend class ExecutionSession;
- std::shared_ptr<EmissionDepUnit> DefiningEDU;
- DenseSet<EmissionDepUnit *> DependantEDUs;
-
LLVM_ABI void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q);
LLVM_ABI void removeQuery(const AsynchronousSymbolQuery &Q);
LLVM_ABI AsynchronousSymbolQueryList
@@ -1778,30 +1765,26 @@ class ExecutionSession {
LLVM_ABI Error OL_notifyResolved(MaterializationResponsibility &MR,
const SymbolMap &Symbols);
- using EDUInfosMap =
- DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo>;
-
- template <typename HandleNewDepFn>
- void propagateExtraEmitDeps(std::deque<JITDylib::EmissionDepUnit *> Worklist,
- EDUInfosMap &EDUInfos,
- HandleNewDepFn HandleNewDep);
- EDUInfosMap simplifyDepGroups(MaterializationResponsibility &MR,
- ArrayRef<SymbolDependenceGroup> EmittedDeps);
- void IL_makeEDUReady(std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries);
- void IL_makeEDUEmitted(std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries);
- bool IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU, JITDylib &DepJD,
- NonOwningSymbolStringPtr DepSym,
- EDUInfosMap &EDUInfos);
-
- static Error makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &ClosedJD);
- static Error makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &BadJD, SymbolNameSet BadDeps);
-
- Expected<JITDylib::AsynchronousSymbolQuerySet>
- IL_emit(MaterializationResponsibility &MR, EDUInfosMap EDUInfos);
+ // FIXME: We should be able to derive FailedSymsForQuery from each query once
+ // we fix how the detach operation works.
+ struct EmitQueries {
+ JITDylib::AsynchronousSymbolQuerySet Updated;
+ JITDylib::AsynchronousSymbolQuerySet Failed;
+ DenseMap<AsynchronousSymbolQuery *, std::shared_ptr<SymbolDependenceMap>>
+ FailedSymsForQuery;
+ };
+
+ WaitingOnGraph::ExternalState
+ IL_getSymbolState(JITDylib *JD, NonOwningSymbolStringPtr Name);
+
+ template <typename UpdateSymbolFn, typename UpdateQueryFn>
+ void IL_collectQueries(JITDylib::AsynchronousSymbolQuerySet &Qs,
+ WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
+ UpdateSymbolFn &&UpdateSymbol,
+ UpdateQueryFn &&UpdateQuery);
+
+ Expected<EmitQueries> IL_emit(MaterializationResponsibility &MR,
+ WaitingOnGraph::SimplifyResult SR);
LLVM_ABI Error OL_notifyEmitted(MaterializationResponsibility &MR,
ArrayRef<SymbolDependenceGroup> EmittedDeps);
@@ -1830,6 +1813,7 @@ class ExecutionSession {
std::vector<ResourceManager *> ResourceManagers;
std::vector<JITDylibSP> JDs;
+ WaitingOnGraph G;
// FIXME: Remove this (and runOutstandingMUs) once the linking layer works
// with callbacks from asynchronous queries.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h
new file mode 100644
index 0000000000000..a5b533351d4d0
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h
@@ -0,0 +1,622 @@
+//===------ WaitingOnGraph.h - ORC symbol dependence graph ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines WaitingOnGraph and related utilities.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H
+#define LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <algorithm>
+
+namespace llvm::orc::detail {
+
+class WaitingOnGraphTest;
+
+/// WaitingOnGraph class template.
+///
+/// This type is intended to provide efficient dependence tracking for Symbols
+/// in an ORC program.
+///
+/// WaitingOnGraph models a directed graph with four partitions:
+/// 1. Not-yet-emitted nodes: Nodes identified as waited-on in an emit
+/// operation.
+/// 2. Emitted nodes: Nodes emitted and waiting on some non-empty set of
+/// other nodes.
+/// 3. Ready nodes: Nodes emitted and not waiting on any other nodes
+/// (either because they weren't waiting on any nodes when they were
+/// emitted, or because all transitively waited-on nodes have since
+/// been emitted).
+/// 4. Failed nodes: Nodes that have been marked as failed-to-emit, and
+/// nodes that were found to transitively wait-on some failed node.
+///
+/// Nodes are added to the graph by *emit* and *fail* operations.
+///
+/// The *emit* operation takes a bipartite *local dependence graph* as an
+/// argument and returns...
+/// a. the set of nodes (both existing and newly added from the local
+/// dependence graph) whose waiting-on set is the empty set, and...
+/// b. the set of newly added nodes that are found to depend on failed
+/// nodes.
+///
+/// The *fail* operation takes a set of failed nodes and returns the set of
+/// Emitted nodes that were waiting on the failed nodes.
+///
+/// The concrete representation adopts several approaches for efficiency:
+///
+/// 1. Only *Emitted* and *Not-yet-emitted* nodes are represented explicitly.
+/// *Ready* and *Failed* nodes are represented by the values returned by the
+/// GetExternalStateFn argument to *emit*.
+///
+/// 2. Labels are (*Container*, *Element*) pairs that are intended to represent
+/// ORC symbols (ORC uses types Container = JITDylib,
+/// Element = NonOwningSymbolStringPtr). The internal representation of the
+/// graph is optimized on the assumption that there are many more Elements
+/// (symbol names) than Containers (JITDylibs) used to construct the labels.
+/// (Consider for example the common case where most JIT'd code is placed in
+/// a single "main" JITDylib).
+///
+/// 3. The data structure stores *SuperNodes* which have multiple labels. This
+/// reduces the number of nodes and edges in the graph in the common case
+/// where many JIT symbols have the same set of dependencies. SuperNodes are
+/// coalesced when their dependence sets become equal.
+///
+/// 4. The *simplify* method can be applied to an initial *local dependence
+/// graph* (as a list of SuperNodes) to eliminate any internal dependence
+/// relationships that would have to be propagated internally by *emit*.
+/// Access to the WaitingOnGraph is assumed to be guarded by a mutex (ORC
+/// will access it from multiple threads) so this allows some pre-processing
+/// to be performed outside the mutex.
+template <typename ContainerIdT, typename ElementIdT> class WaitingOnGraph {
+ friend class WaitingOnGraphTest;
+
+public:
+ using ContainerId = ContainerIdT;
+ using ElementId = ElementIdT;
+ using ElementSet = DenseSet<ElementId>;
+ using ContainerElementsMap = DenseMap<ContainerId, ElementSet>;
+
+ class SuperNode {
+ friend class WaitingOnGraph;
+ friend class WaitingOnGraphTest;
+
+ public:
+ SuperNode(ContainerElementsMap Defs, ContainerElementsMap Deps)
+ : Defs(std::move(Defs)), Deps(std::move(Deps)) {}
+ ContainerElementsMap &defs() { return Defs; }
+ const ContainerElementsMap &defs() const { return Defs; }
+ ContainerElementsMap &deps() { return Deps; }
+ const ContainerElementsMap &deps() const { return Deps; }
+
+ private:
+ ContainerElementsMap Defs;
+ ContainerElementsMap Deps;
+ };
+
+private:
+ using ElemToSuperNodeMap =
+ DenseMap<ContainerId, DenseMap<ElementId, SuperNode *>>;
+
+ using SuperNodeDepsMap = DenseMap<SuperNode *, DenseSet<SuperNode *>>;
+
+ class Coalescer {
+ public:
+ std::unique_ptr<SuperNode> addOrCreateSuperNode(ContainerElementsMap Defs,
+ ContainerElementsMap Deps) {
+ auto H = getHash(Deps);
+ if (auto *ExistingSN = findCanonicalSuperNode(H, Deps)) {
+ for (auto &[Container, Elems] : Defs) {
+ auto &DstCElems = ExistingSN->Defs[Container];
+ [[maybe_unused]] size_t ExpectedSize =
+ DstCElems.size() + Elems.size();
+ DstCElems.insert(Elems.begin(), Elems.end());
+ assert(DstCElems.size() == ExpectedSize);
+ }
+ return nullptr;
+ }
+
+ auto NewSN =
+ std::make_unique<SuperNode>(std::move(Defs), std::move(Deps));
+ CanonicalSNs[H].push_back(NewSN.get());
+ return NewSN;
+ }
+
+ void coalesce(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ ElemToSuperNodeMap &ElemToSN) {
+ for (size_t I = 0; I != SNs.size();) {
+ auto &SN = SNs[I];
+ auto H = getHash(SN->Deps);
+ if (auto *CanonicalSN = findCanonicalSuperNode(H, SN->Deps)) {
+ for (auto &[Container, Elems] : SN->Defs) {
+ CanonicalSN->Defs[Container].insert(Elems.begin(), Elems.end());
+ auto &ContainerElemToSN = ElemToSN[Container];
+ for (auto &Elem : Elems)
+ ContainerElemToSN[Elem] = CanonicalSN;
+ }
+ std::swap(SN, SNs.back());
+ SNs.pop_back();
+ } else {
+ CanonicalSNs[H].push_back(SN.get());
+ ++I;
+ }
+ }
+ }
+
+ template <typename Pred> void remove(Pred &&Remove) {
+ for (auto &[Hash, SNs] : CanonicalSNs) {
+ bool Found = false;
+ for (size_t I = 0; I != SNs.size(); ++I) {
+ if (Remove(SNs[I])) {
+ std::swap(SNs[I], SNs.back());
+ SNs.pop_back();
+ Found = true;
+ break;
+ }
+ }
+ if (Found) {
+ if (SNs.empty())
+ CanonicalSNs.erase(Hash);
+ break;
+ }
+ }
+ }
+
+ private:
+ hash_code getHash(const ContainerElementsMap &M) {
+ SmallVector<ContainerId> SortedContainers;
+ SortedContainers.reserve(M.size());
+ for (auto &[Container, Elems] : M)
+ SortedContainers.push_back(Container);
+ llvm::sort(SortedContainers);
+ hash_code Hash(0);
+ for (auto &Container : SortedContainers) {
+ auto &ContainerElems = M.at(Container);
+ SmallVector<ElementId> SortedElems(ContainerElems.begin(),
+ ContainerElems.end());
+ llvm::sort(SortedElems);
+ Hash = hash_combine(
+ Hash, Container,
+ hash_combine_range(SortedElems.begin(), SortedElems.end()));
+ }
+ return Hash;
+ }
+
+ SuperNode *findCanonicalSuperNode(hash_code H,
+ const ContainerElementsMap &M) {
+ for (auto *SN : CanonicalSNs[H])
+ if (SN->Deps == M)
+ return SN;
+ return nullptr;
+ }
+
+ DenseMap<hash_code, SmallVector<SuperNode *>> CanonicalSNs;
+ };
+
+public:
+ /// Build SuperNodes from (definition-set, dependence-set) pairs.
+ ///
+ /// Coalesces definition-sets with identical dependence-sets.
+ class SuperNodeBuilder {
+ public:
+ void add(ContainerElementsMap Defs, ContainerElementsMap Deps) {
+ if (Defs.empty())
+ return;
+ // Remove any self-reference.
+ SmallVector<ContainerId> ToRemove;
+ for (auto &[Container, Elems] : Defs) {
+ assert(!Elems.empty() && "Defs for container must not be empty");
+ auto I = Deps.find(Container);
+ if (I == Deps.end())
+ continue;
+ auto &DepsForContainer = I->second;
+ for (auto &Elem : Elems)
+ DepsForContainer.erase(Elem);
+ if (DepsForContainer.empty())
+ ToRemove.push_back(Container);
+ }
+ for (auto &Container : ToRemove)
+ Deps.erase(Container);
+ if (auto SN = C.addOrCreateSuperNode(std::move(Defs), std::move(Deps)))
+ SNs.push_back(std::move(SN));
+ }
+ std::vector<std::unique_ptr<SuperNode>> takeSuperNodes() {
+ return std::move(SNs);
+ }
+
+ private:
+ Coalescer C;
+ std::vector<std::unique_ptr<SuperNode>> SNs;
+ };
+
+ class SimplifyResult {
+ friend class WaitingOnGraph;
+ friend class WaitingOnGraphTest;
+
+ public:
+ const std::vector<std::unique_ptr<SuperNode>> &superNodes() const {
+ return SNs;
+ }
+
+ private:
+ SimplifyResult(std::vector<std::unique_ptr<SuperNode>> SNs,
+ ElemToSuperNodeMap ElemToSN)
+ : SNs(std::move(SNs)), ElemToSN(std::move(ElemToSN)) {}
+ std::vector<std::unique_ptr<SuperNode>> SNs;
+ ElemToSuperNodeMap ElemToSN;
+ };
+
+ /// Preprocess a list of SuperNodes to remove all intra-SN dependencies.
+ static SimplifyResult simplify(std::vector<std::unique_ptr<SuperNode>> SNs) {
+ // Build ElemToSN map.
+ ElemToSuperNodeMap ElemToSN;
+ for (auto &SN : SNs) {
+ for (auto &[Container, Elements] : SN->Defs) {
+ auto &ContainerElemToSN = ElemToSN[Container];
+ for (auto &E : Elements)
+ ContainerElemToSN[E] = SN.get();
+ }
+ }
+
+ SuperNodeDepsMap SuperNodeDeps;
+ hoistDeps(SuperNodeDeps, SNs, ElemToSN);
+ propagateSuperNodeDeps(SuperNodeDeps);
+ sinkDeps(SNs, SuperNodeDeps);
+
+ // Pre-coalesce nodes.
+ Coalescer().coalesce(SNs, ElemToSN);
+
+ return {std::move(SNs), std::move(ElemToSN)};
+ }
+
+ struct EmitResult {
+ std::vector<std::unique_ptr<SuperNode>> Ready;
+ std::vector<std::unique_ptr<SuperNode>> Failed;
+ };
+
+ enum class ExternalState { None, Ready, Failed };
+
+ /// Add the given SuperNodes to the graph, returning any SuperNodes that
+ /// move to the Ready or Failed states as a result.
+ /// The GetExternalState function is used to represent SuperNodes that have
+ /// already become Ready or Failed (since such nodes are not explicitly
+ /// represented in the graph).
+ template <typename GetExternalStateFn>
+ EmitResult emit(SimplifyResult SR, GetExternalStateFn &&GetExternalState) {
+ auto NewSNs = std::move(SR.SNs);
+ auto ElemToNewSN = std::move(SR.ElemToSN);
+
+ // First process any dependencies on nodes with external state.
+ auto FailedSNs = processExternalDeps(NewSNs, GetExternalState);
+
+ // Collect the PendingSNs whose dep sets are about to be modified.
+ std::vector<std::unique_ptr<SuperNode>> ModifiedPendingSNs;
+ for (size_t I = 0; I != PendingSNs.size();) {
+ auto &SN = PendingSNs[I];
+ bool Remove = false;
+ for (auto &[Container, Elems] : SN->Deps) {
+ auto I = ElemToNewSN.find(Container);
+ if (I == ElemToNewSN.end())
+ continue;
+ for (auto Elem : Elems) {
+ if (I->second.contains(Elem)) {
+ Remove = true;
+ break;
+ }
+ }
+ if (Remove)
+ break;
+ }
+ if (Remove) {
+ ModifiedPendingSNs.push_back(std::move(SN));
+ std::swap(SN, PendingSNs.back());
+ PendingSNs.pop_back();
+ } else
+ ++I;
+ }
+
+ // Remove cycles from the graphs.
+ SuperNodeDepsMap SuperNodeDeps;
+ hoistDeps(SuperNodeDeps, ModifiedPendingSNs, ElemToNewSN);
+
+ CoalesceToPendingSNs.remove(
+ [&](SuperNode *SN) { return SuperNodeDeps.count(SN); });
+
+ hoistDeps(SuperNodeDeps, NewSNs, ElemToPendingSN);
+ propagateSuperNodeDeps(SuperNodeDeps);
+ sinkDeps(NewSNs, SuperNodeDeps);
+ sinkDeps(ModifiedPendingSNs, SuperNodeDeps);
+
+ // Process supernodes. Pending first, since we'll update PendingSNs when we
+ // incorporate NewSNs.
+ std::vector<std::unique_ptr<SuperNode>> ReadyNodes, FailedNodes;
+ processReadyOrFailed(ModifiedPendingSNs, ReadyNodes, FailedNodes,
+ SuperNodeDeps, ElemToPendingSN, FailedSNs);
+ processReadyOrFailed(NewSNs, ReadyNodes, FailedNodes, SuperNodeDeps,
+ ElemToNewSN, FailedSNs);
+
+ CoalesceToPendingSNs.coalesce(ModifiedPendingSNs, ElemToPendingSN);
+ CoalesceToPendingSNs.coalesce(NewSNs, ElemToPendingSN);
+
+ // Integrate remaining ModifiedPendingSNs and NewSNs into PendingSNs.
+ for (auto &SN : ModifiedPendingSNs)
+ PendingSNs.push_back(std::move(SN));
+
+ // Update ElemToPendingSN for the remaining elements.
+ for (auto &SN : NewSNs) {
+ for (auto &[Container, Elems] : SN->Defs) {
+ auto &Row = ElemToPendingSN[Container];
+ for (auto &Elem : Elems)
+ Row[Elem] = SN.get();
+ }
+ PendingSNs.push_back(std::move(SN));
+ }
+
+ return {std::move(ReadyNodes), std::move(FailedNodes)};
+ }
+
+ /// Identify the given symbols as Failed.
+ /// The elements of the Failed map will not be included in the returned
+ /// result, so clients should take whatever actions are needed to mark
+ /// this as failed in their external representation.
+ std::vector<std::unique_ptr<SuperNode>>
+ fail(const ContainerElementsMap &Failed) {
+ std::vector<std::unique_ptr<SuperNode>> FailedSNs;
+
+ for (size_t I = 0; I != PendingSNs.size();) {
+ auto &PendingSN = PendingSNs[I];
+ bool FailPendingSN = false;
+ for (auto &[Container, Elems] : PendingSN->Deps) {
+ if (FailPendingSN)
+ break;
+ auto I = Failed.find(Container);
+ if (I == Failed.end())
+ continue;
+ for (auto &Elem : Elems) {
+ if (I->second.count(Elem)) {
+ FailPendingSN = true;
+ break;
+ }
+ }
+ }
+ if (FailPendingSN) {
+ FailedSNs.push_back(std::move(PendingSN));
+ PendingSN = std::move(PendingSNs.back());
+ PendingSNs.pop_back();
+ } else
+ ++I;
+ }
+
+ for (auto &SN : FailedSNs) {
+ CoalesceToPendingSNs.remove(
+ [&](SuperNode *SNC) { return SNC == SN.get(); });
+ for (auto &[Container, Elems] : SN->Defs) {
+ assert(ElemToPendingSN.count(Container));
+ auto &CElems = ElemToPendingSN[Container];
+ for (auto &Elem : Elems)
+ CElems.erase(Elem);
+ if (CElems.empty())
+ ElemToPendingSN.erase(Container);
+ }
+ }
+
+ return FailedSNs;
+ }
+
+ bool validate(raw_ostream &Log) {
+ bool AllGood = true;
+ auto ErrLog = [&]() -> raw_ostream & {
+ AllGood = false;
+ return Log;
+ };
+
+ size_t DefCount = 0;
+ for (auto &PendingSN : PendingSNs) {
+ if (PendingSN->Deps.empty())
+ ErrLog() << "Pending SN " << PendingSN.get() << " has empty dep set.\n";
+ else {
+ bool BadElem = false;
+ for (auto &[Container, Elems] : PendingSN->Deps) {
+ auto I = ElemToPendingSN.find(Container);
+ if (I == ElemToPendingSN.end())
+ continue;
+ if (Elems.empty())
+ ErrLog() << "Pending SN " << PendingSN.get()
+ << " has dependence map entry for " << Container
+ << " with empty element set.\n";
+ for (auto &Elem : Elems) {
+ if (I->second.count(Elem)) {
+ ErrLog() << "Pending SN " << PendingSN.get()
+ << " has dependence on emitted element ( " << Container
+ << ", " << Elem << ")\n";
+ BadElem = true;
+ break;
+ }
+ }
+ if (BadElem)
+ break;
+ }
+ }
+
+ for (auto &[Container, Elems] : PendingSN->Defs) {
+ if (Elems.empty())
+ ErrLog() << "Pending SN " << PendingSN.get()
+ << " has def map entry for " << Container
+ << " with empty element set.\n";
+ DefCount += Elems.size();
+ auto I = ElemToPendingSN.find(Container);
+ if (I == ElemToPendingSN.end())
+ ErrLog() << "Pending SN " << PendingSN.get() << " has "
+ << Elems.size() << " defs in container " << Container
+ << " not covered by ElemsToPendingSN.\n";
+ else {
+ for (auto &Elem : Elems) {
+ auto J = I->second.find(Elem);
+ if (J == I->second.end())
+ ErrLog() << "Pending SN " << PendingSN.get() << " has element ("
+ << Container << ", " << Elem
+ << ") not covered by ElemsToPendingSN.\n";
+ else if (J->second != PendingSN.get())
+ ErrLog() << "ElemToPendingSN value invalid for (" << Container
+ << ", " << Elem << ")\n";
+ }
+ }
+ }
+ }
+
+ size_t DefCount2 = 0;
+ for (auto &[Container, Elems] : ElemToPendingSN)
+ DefCount2 += Elems.size();
+
+ assert(DefCount2 >= DefCount);
+ if (DefCount2 != DefCount)
+ ErrLog() << "ElemToPendingSN contains extra elements.\n";
+
+ return AllGood;
+ }
+
+private:
+ // Replace individual dependencies with supernode dependencies.
+ //
+ // For all dependencies in SNs, if the corresponding node is defined in
+ // ElemToSN then remove the individual dependency and add the record the
+ // dependency on the corresponding supernode in SuperNodeDeps.
+ static void hoistDeps(SuperNodeDepsMap &SuperNodeDeps,
+ std::vector<std::unique_ptr<SuperNode>> &SNs,
+ ElemToSuperNodeMap &ElemToSN) {
+ for (auto &SN : SNs) {
+ auto &SNDeps = SuperNodeDeps[SN.get()];
+ for (auto &[DefContainer, DefElems] : ElemToSN) {
+ auto I = SN->Deps.find(DefContainer);
+ if (I == SN->Deps.end())
+ continue;
+ for (auto &[DefElem, DefSN] : DefElems)
+ if (I->second.erase(DefElem))
+ SNDeps.insert(DefSN);
+ if (I->second.empty())
+ SN->Deps.erase(I);
+ }
+ }
+ }
+
+ // Compute transitive closure of deps for each node.
+ static void propagateSuperNodeDeps(SuperNodeDepsMap &SuperNodeDeps) {
+ for (auto &[SN, Deps] : SuperNodeDeps) {
+ DenseSet<SuperNode *> Reachable({SN});
+ SmallVector<SuperNode *> Worklist(Deps.begin(), Deps.end());
+
+ while (!Worklist.empty()) {
+ auto *DepSN = Worklist.pop_back_val();
+ if (!Reachable.insert(DepSN).second)
+ continue;
+ auto I = SuperNodeDeps.find(DepSN);
+ if (I == SuperNodeDeps.end())
+ continue;
+ for (auto *DepSNDep : I->second)
+ Worklist.push_back(DepSNDep);
+ }
+
+ Deps = std::move(Reachable);
+ }
+ }
+
+ // Sink SuperNode dependencies back to dependencies on individual nodes.
+ static void sinkDeps(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ SuperNodeDepsMap &SuperNodeDeps) {
+ for (auto &SN : SNs) {
+ auto I = SuperNodeDeps.find(SN.get());
+ if (I == SuperNodeDeps.end())
+ continue;
+
+ for (auto *DepSN : I->second)
+ for (auto &[Container, Elems] : DepSN->Deps)
+ SN->Deps[Container].insert(Elems.begin(), Elems.end());
+ }
+ }
+
+ template <typename GetExternalStateFn>
+ static std::vector<SuperNode *>
+ processExternalDeps(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ GetExternalStateFn &GetExternalState) {
+ std::vector<SuperNode *> FailedSNs;
+ for (auto &SN : SNs) {
+ bool SNHasError = false;
+ SmallVector<ContainerId> ContainersToRemove;
+ for (auto &[Container, Elems] : SN->Deps) {
+ SmallVector<ElementId> ElemToRemove;
+ for (auto &Elem : Elems) {
+ switch (GetExternalState(Container, Elem)) {
+ case ExternalState::None:
+ break;
+ case ExternalState::Ready:
+ ElemToRemove.push_back(Elem);
+ break;
+ case ExternalState::Failed:
+ ElemToRemove.push_back(Elem);
+ SNHasError = true;
+ break;
+ }
+ }
+ for (auto &Elem : ElemToRemove)
+ Elems.erase(Elem);
+ if (Elems.empty())
+ ContainersToRemove.push_back(Container);
+ }
+ for (auto &Container : ContainersToRemove)
+ SN->Deps.erase(Container);
+ if (SNHasError)
+ FailedSNs.push_back(SN.get());
+ }
+
+ return FailedSNs;
+ }
+
+ void processReadyOrFailed(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ std::vector<std::unique_ptr<SuperNode>> &Ready,
+ std::vector<std::unique_ptr<SuperNode>> &Failed,
+ SuperNodeDepsMap &SuperNodeDeps,
+ ElemToSuperNodeMap &ElemToSNs,
+ std::vector<SuperNode *> FailedSNs) {
+ for (size_t I = 0; I != SNs.size();) {
+ auto &SN = SNs[I];
+
+ bool SNFailed = false;
+ assert(SuperNodeDeps.count(SN.get()));
+ auto &SNSuperNodeDeps = SuperNodeDeps[SN.get()];
+ for (auto *FailedSN : FailedSNs) {
+ if (FailedSN == SN.get() || SNSuperNodeDeps.count(FailedSN)) {
+ SNFailed = true;
+ break;
+ }
+ }
+
+ bool SNReady = SN->Deps.empty();
+
+ if (SNReady || SNFailed) {
+ auto &NodeList = SNFailed ? Failed : Ready;
+ NodeList.push_back(std::move(SN));
+ std::swap(SN, SNs.back());
+ SNs.pop_back();
+ } else
+ ++I;
+ }
+ }
+
+ std::vector<std::unique_ptr<SuperNode>> PendingSNs;
+ ElemToSuperNodeMap ElemToPendingSN;
+ Coalescer CoalesceToPendingSNs;
+};
+
+} // namespace llvm::orc::detail
+
+#endif // LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index f47b7ecdcc7bb..091c94347bd06 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -1173,39 +1173,7 @@ void JITDylib::dump(raw_ostream &OS) {
<< " pending queries: { ";
for (const auto &Q : KV.second.pendingQueries())
OS << Q.get() << " (" << Q->getRequiredState() << ") ";
- OS << "}\n Defining EDU: ";
- if (KV.second.DefiningEDU) {
- OS << KV.second.DefiningEDU.get() << " { ";
- for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
- OS << Name << " ";
- OS << "}\n";
- OS << " Dependencies:\n";
- if (!KV.second.DefiningEDU->Dependencies.empty()) {
- for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
- OS << " " << DepJD->getName() << ": [ ";
- for (auto &Dep : Deps)
- OS << Dep << " ";
- OS << "]\n";
- }
- } else
- OS << " none\n";
- } else
- OS << "none\n";
- OS << " Dependant EDUs:\n";
- if (!KV.second.DependantEDUs.empty()) {
- for (auto &DependantEDU : KV.second.DependantEDUs) {
- OS << " " << DependantEDU << ": "
- << DependantEDU->JD->getName() << " { ";
- for (auto &[Name, Flags] : DependantEDU->Symbols)
- OS << Name << " ";
- OS << "}\n";
- }
- } else
- OS << " none\n";
- assert((Symbols[KV.first].getState() != SymbolState::Ready ||
- (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
- !KV.second.DependantEDUs.empty())) &&
- "Stale materializing info entry");
+ OS << "}\n";
}
});
}
@@ -1967,9 +1935,6 @@ bool ExecutionSession::verifySessionState(Twine Phase) {
return runSessionLocked([&]() {
bool AllOk = true;
- // We'll collect these and verify them later to avoid redundant checks.
- DenseSet<JITDylib::EmissionDepUnit *> EDUsToCheck;
-
for (auto &JD : JDs) {
auto LogFailure = [&]() -> raw_fd_ostream & {
@@ -2063,89 +2028,10 @@ bool ExecutionSession::verifySessionState(Twine Phase) {
<< " has stale or misordered queries.\n";
}
}
-
- // If there's a DefiningEDU then check that...
- // 1. The JD matches.
- // 2. The symbol is in the EDU's Symbols map.
- // 3. The symbol table entry is in the Emitted state.
- if (MII.DefiningEDU) {
-
- EDUsToCheck.insert(MII.DefiningEDU.get());
-
- if (MII.DefiningEDU->JD != JD.get()) {
- LogFailure() << "symbol " << Sym
- << " has DefiningEDU with incorrect JD"
- << (llvm::is_contained(JDs, MII.DefiningEDU->JD)
- ? " (JD not currently in ExecutionSession"
- : "")
- << "\n";
- }
-
- if (SymItr->second.getState() != SymbolState::Emitted) {
- LogFailure()
- << "symbol " << Sym
- << " has DefiningEDU, but is not in Emitted state.\n";
- }
- }
-
- // Check that JDs for any DependantEDUs are also in the session --
- // that guarantees that we'll also visit them during this loop.
- for (auto &DepEDU : MII.DependantEDUs) {
- if (!llvm::is_contained(JDs, DepEDU->JD)) {
- LogFailure() << "symbol " << Sym << " has DependantEDU "
- << (void *)DepEDU << " with JD (" << DepEDU->JD
- << ") that isn't in ExecutionSession.\n";
- }
- }
}
}
}
- // Check EDUs.
- for (auto *EDU : EDUsToCheck) {
- assert(EDU->JD->State == JITDylib::Open && "EDU->JD is not Open");
-
- auto LogFailure = [&]() -> raw_fd_ostream & {
- AllOk = false;
- auto &Stream = errs();
- Stream << "In EDU defining " << EDU->JD->getName() << ": { ";
- for (auto &[Sym, Flags] : EDU->Symbols)
- Stream << Sym << " ";
- Stream << "}, ";
- return Stream;
- };
-
- if (EDU->Symbols.empty())
- LogFailure() << "no symbols defined.\n";
- else {
- for (auto &[Sym, Flags] : EDU->Symbols) {
- if (!Sym)
- LogFailure() << "null symbol defined.\n";
- else {
- if (!EDU->JD->Symbols.count(SymbolStringPtr(Sym))) {
- LogFailure() << "symbol " << Sym
- << " isn't present in JD's symbol table.\n";
- }
- }
- }
- }
-
- for (auto &[DepJD, Symbols] : EDU->Dependencies) {
- if (!llvm::is_contained(JDs, DepJD)) {
- LogFailure() << "dependant symbols listed for JD that isn't in "
- "ExecutionSession.\n";
- } else {
- for (auto &DepSym : Symbols) {
- if (!DepJD->Symbols.count(SymbolStringPtr(DepSym))) {
- LogFailure()
- << "dependant symbol " << DepSym
- << " does not appear in symbol table for dependant JD "
- << DepJD->getName() << ".\n";
- }
- }
- }
- }
- }
return AllOk;
});
@@ -2917,359 +2803,64 @@ Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
return MR.JD.resolve(MR, Symbols);
}
-template <typename HandleNewDepFn>
-void ExecutionSession::propagateExtraEmitDeps(
- std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
- HandleNewDepFn HandleNewDep) {
-
- // Iterate to a fixed-point to propagate extra-emit dependencies through the
- // EDU graph.
- while (!Worklist.empty()) {
- auto &EDU = *Worklist.front();
- Worklist.pop_front();
-
- assert(EDUInfos.count(&EDU) && "No info entry for EDU");
- auto &EDUInfo = EDUInfos[&EDU];
-
- // Propagate new dependencies to users.
- for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
-
- // UserEDUInfo only present if UserEDU has its own users.
- JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
- {
- auto UserEDUInfoItr = EDUInfos.find(UserEDU);
- if (UserEDUInfoItr != EDUInfos.end())
- UserEDUInfo = &UserEDUInfoItr->second;
- }
-
- for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
- auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
- DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
- for (auto Dep : Deps) {
- if (UserEDUDepsForJD.insert(Dep).second) {
- HandleNewDep(*UserEDU, *DepJD, Dep);
- if (UserEDUInfo) {
- if (!UserEDUNewDepsForJD) {
- // If UserEDU has no new deps then it's not in the worklist
- // yet, so add it.
- if (UserEDUInfo->NewDeps.empty())
- Worklist.push_back(UserEDU);
- UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
- }
- // Add (DepJD, Dep) to NewDeps.
- UserEDUNewDepsForJD->insert(Dep);
- }
- }
+WaitingOnGraph::ExternalState
+ExecutionSession::IL_getSymbolState(JITDylib *JD,
+ NonOwningSymbolStringPtr Name) {
+ if (JD->State != JITDylib::Open)
+ return WaitingOnGraph::ExternalState::Failed;
+
+ auto I = JD->Symbols.find_as(Name);
+
+ // FIXME: Can we eliminate this possibility if we support query binding?
+ if (I == JD->Symbols.end())
+ return WaitingOnGraph::ExternalState::Failed;
+
+ if (I->second.getFlags().hasError())
+ return WaitingOnGraph::ExternalState::Failed;
+
+ if (I->second.getState() == SymbolState::Ready)
+ return WaitingOnGraph::ExternalState::Ready;
+
+ return WaitingOnGraph::ExternalState::None;
+}
+
+template <typename UpdateSymbolFn, typename UpdateQueryFn>
+void ExecutionSession::IL_collectQueries(
+ JITDylib::AsynchronousSymbolQuerySet &Qs,
+ WaitingOnGraph::ContainerElementsMap &QualifiedSymbols,
+ UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) {
+
+ for (auto &[JD, Symbols] : QualifiedSymbols) {
+ // IL_emit and JITDylib removal are synchronized by the session lock.
+ // Since JITDylib removal removes any contained nodes from the
+ // WaitingOnGraph, we should be able to assert that all nodes in the
+ // WaitingOnGraph have not been removed.
+ assert(JD->State == JITDylib::Open &&
+ "WaitingOnGraph includes definition in defunct JITDylib");
+ for (auto &Symbol : Symbols) {
+ // Update symbol table.
+ auto I = JD->Symbols.find_as(Symbol);
+ assert(I != JD->Symbols.end() &&
+ "Failed Symbol missing from JD symbol table");
+ auto &Entry = I->second;
+ UpdateSymbol(Entry);
+
+ // Collect queries.
+ auto J = JD->MaterializingInfos.find_as(Symbol);
+ if (J != JD->MaterializingInfos.end()) {
+ for (auto &Q : J->second.takeAllPendingQueries()) {
+ UpdateQuery(*Q, *JD, Symbol, Entry);
+ Qs.insert(std::move(Q));
}
+ JD->MaterializingInfos.erase(J);
}
}
-
- EDUInfo.NewDeps.clear();
}
}
-// Note: This method modifies the emitted set.
-ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
- MaterializationResponsibility &MR,
- ArrayRef<SymbolDependenceGroup> EmittedDeps) {
-
- auto &TargetJD = MR.getTargetJITDylib();
-
- // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
- // Symbol -> EmissionDepUnit mappings.
- DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
- EDUInfos.reserve(EmittedDeps.size());
- DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
- for (auto &DG : EmittedDeps) {
- assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
-
- // Skip empty EDUs.
- if (DG.Dependencies.empty())
- continue;
-
- auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
- auto &EDUInfo = EDUInfos[TmpEDU.get()];
- EDUInfo.EDU = std::move(TmpEDU);
- for (const auto &Symbol : DG.Symbols) {
- NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
- assert(!EDUForSymbol.count(NonOwningSymbol) &&
- "Symbol should not appear in more than one SymbolDependenceGroup");
- assert(MR.getSymbols().count(Symbol) &&
- "Symbol in DepGroups not in the emitted set");
- auto NewlyEmittedItr = MR.getSymbols().find(Symbol);
- EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
- EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
- }
- }
-
- // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
- {
- DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
- for (auto &[Sym, Flags] : MR.getSymbols()) {
- if (!EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)))
- ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
- }
- if (!ResidualSymbolFlags.empty()) {
- auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
- ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
- auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
- ResidualEDUInfo.EDU = std::move(ResidualEDU);
-
- // If the residual EDU is the only one then bail out early.
- if (EDUInfos.size() == 1)
- return EDUInfos;
-
- // Otherwise add the residual EDU to the EDUForSymbol map.
- for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
- EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
- }
- }
-
-#ifndef NDEBUG
- assert(EDUForSymbol.size() == MR.getSymbols().size() &&
- "MR symbols not fully covered by EDUs?");
- for (auto &[Sym, Flags] : MR.getSymbols()) {
- assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
- "Sym in MR not covered by EDU");
- }
-#endif // NDEBUG
-
- // 3. Use the DepGroups array to build a graph of dependencies between
- // EmissionDepUnits in this finalization. We want to remove these
- // intra-finalization uses, propagating dependencies on symbols outside
- // this finalization. Add EDUs to the worklist.
- for (auto &DG : EmittedDeps) {
-
- // Skip SymbolDependenceGroups with no dependencies.
- if (DG.Dependencies.empty())
- continue;
-
- assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
- "No EDU for DG");
- auto &EDU =
- *EDUForSymbol.find(NonOwningSymbolStringPtr(*DG.Symbols.begin()))
- ->second;
-
- for (auto &[DepJD, Deps] : DG.Dependencies) {
- DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
-
- assert(!Deps.empty() && "Dependence set for DepJD is empty");
-
- if (DepJD != &TargetJD) {
- // DepJD is some other JITDylib.There can't be any intra-finalization
- // edges here, so just skip.
- for (auto &Dep : Deps)
- NewDepsForJD.insert(NonOwningSymbolStringPtr(Dep));
- } else {
- // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
- // skipping any and recording the intra-finalization use instead.
- for (auto &Dep : Deps) {
- NonOwningSymbolStringPtr NonOwningDep(Dep);
- auto I = EDUForSymbol.find(NonOwningDep);
- if (I == EDUForSymbol.end()) {
- if (!MR.getSymbols().count(Dep))
- NewDepsForJD.insert(NonOwningDep);
- continue;
- }
-
- if (I->second != &EDU)
- EDUInfos[I->second].IntraEmitUsers.insert(&EDU);
- }
- }
-
- if (!NewDepsForJD.empty())
- EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
- }
- }
-
- // 4. Build the worklist.
- std::deque<JITDylib::EmissionDepUnit *> Worklist;
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- // If this EDU has extra-finalization dependencies and intra-finalization
- // users then add it to the worklist.
- if (!EDU->Dependencies.empty()) {
- auto I = EDUInfos.find(EDU);
- if (I != EDUInfos.end()) {
- auto &EDUInfo = I->second;
- if (!EDUInfo.IntraEmitUsers.empty()) {
- EDUInfo.NewDeps = EDU->Dependencies;
- Worklist.push_back(EDU);
- }
- }
- }
- }
-
- // 4. Propagate dependencies through the EDU graph.
- propagateExtraEmitDeps(
- Worklist, EDUInfos,
- [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
-
- return EDUInfos;
-}
-
-void ExecutionSession::IL_makeEDUReady(
- std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries) {
-
- // The symbols for this EDU are ready.
- auto &JD = *EDU->JD;
-
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
- "JD does not have an entry for Sym");
- auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
-
- assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
- Entry.getState() == SymbolState::Materializing) ||
- Entry.getState() == SymbolState::Resolved ||
- Entry.getState() == SymbolState::Emitted) &&
- "Emitting from state other than Resolved");
-
- Entry.setState(SymbolState::Ready);
-
- auto MII = JD.MaterializingInfos.find(SymbolStringPtr(Sym));
-
- // Check for pending queries.
- if (MII == JD.MaterializingInfos.end())
- continue;
- auto &MI = MII->second;
-
- for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) {
- Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
- if (Q->isComplete())
- Queries.insert(Q);
- Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
- }
-
- JD.MaterializingInfos.erase(MII);
- }
-
- JD.shrinkMaterializationInfoMemory();
-}
-
-void ExecutionSession::IL_makeEDUEmitted(
- std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
- JITDylib::AsynchronousSymbolQuerySet &Queries) {
-
- // The symbols for this EDU are emitted, but not ready.
- auto &JD = *EDU->JD;
-
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
- "JD does not have an entry for Sym");
- auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
-
- assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
- Entry.getState() == SymbolState::Materializing) ||
- Entry.getState() == SymbolState::Resolved ||
- Entry.getState() == SymbolState::Emitted) &&
- "Emitting from state other than Resolved");
-
- if (Entry.getState() == SymbolState::Emitted) {
- // This was already emitted, so we can skip the rest of this loop.
-#ifndef NDEBUG
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
- "JD does not have an entry for Sym");
- auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
- assert(Entry.getState() == SymbolState::Emitted &&
- "Symbols for EDU in inconsistent state");
- assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
- "Emitted symbol has no MI");
- auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
- assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
- "Already-emitted symbol has waiting-on-emitted queries");
- }
-#endif // NDEBUG
- break;
- }
-
- Entry.setState(SymbolState::Emitted);
- auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
- MI.DefiningEDU = EDU;
-
- for (auto &Q : MI.takeQueriesMeeting(SymbolState::Emitted)) {
- Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
- if (Q->isComplete())
- Queries.insert(Q);
- }
- }
-
- for (auto &[DepJD, Deps] : EDU->Dependencies) {
- for (auto &Dep : Deps)
- DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
- EDU.get());
- }
-}
-
-/// Removes the given dependence from EDU. If EDU's dependence set becomes
-/// empty then this function adds an entry for it to the EDUInfos map.
-/// Returns true if a new EDUInfosMap entry is added.
-bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
- JITDylib &DepJD,
- NonOwningSymbolStringPtr DepSym,
- EDUInfosMap &EDUInfos) {
- assert(EDU.Dependencies.count(&DepJD) &&
- "JD does not appear in Dependencies of DependantEDU");
- assert(EDU.Dependencies[&DepJD].count(DepSym) &&
- "Symbol does not appear in Dependencies of DependantEDU");
- auto &JDDeps = EDU.Dependencies[&DepJD];
- JDDeps.erase(DepSym);
- if (JDDeps.empty()) {
- EDU.Dependencies.erase(&DepJD);
- if (EDU.Dependencies.empty()) {
- // If the dependencies set has become empty then EDU _may_ be ready
- // (we won't know for sure until we've propagated the extra-emit deps).
- // Create an EDUInfo for it (if it doesn't have one already) so that
- // it'll be visited after propagation.
- auto &DepEDUInfo = EDUInfos[&EDU];
- if (!DepEDUInfo.EDU) {
- assert(EDU.JD->Symbols.count(
- SymbolStringPtr(EDU.Symbols.begin()->first)) &&
- "Missing symbol entry for first symbol in EDU");
- auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
- SymbolStringPtr(EDU.Symbols.begin()->first));
- assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
- "Missing MI for first symbol in DependantEDU");
- DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
- return true;
- }
- }
- }
- return false;
-}
-
-Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &ClosedJD) {
- SymbolNameSet FailedSymbols;
- for (auto &[Sym, Flags] : EDU.Symbols)
- FailedSymbols.insert(SymbolStringPtr(Sym));
- SymbolDependenceMap BadDeps;
- for (auto &Dep : EDU.Dependencies[&ClosedJD])
- BadDeps[&ClosedJD].insert(SymbolStringPtr(Dep));
- return make_error<UnsatisfiedSymbolDependencies>(
- ClosedJD.getExecutionSession().getSymbolStringPool(), EDU.JD,
- std::move(FailedSymbols), std::move(BadDeps),
- ClosedJD.getName() + " is closed");
-}
-
-Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
- JITDylib &BadJD,
- SymbolNameSet BadDeps) {
- SymbolNameSet FailedSymbols;
- for (auto &[Sym, Flags] : EDU.Symbols)
- FailedSymbols.insert(SymbolStringPtr(Sym));
- SymbolDependenceMap BadDepsMap;
- BadDepsMap[&BadJD] = std::move(BadDeps);
- return make_error<UnsatisfiedSymbolDependencies>(
- BadJD.getExecutionSession().getSymbolStringPool(), &BadJD,
- std::move(FailedSymbols), std::move(BadDepsMap),
- "dependencies removed or in error state");
-}
-
-Expected<JITDylib::AsynchronousSymbolQuerySet>
+Expected<ExecutionSession::EmitQueries>
ExecutionSession::IL_emit(MaterializationResponsibility &MR,
- EDUInfosMap EDUInfos) {
+ WaitingOnGraph::SimplifyResult SR) {
if (MR.RT->isDefunct())
return make_error<ResourceTrackerDefunct>(MR.RT);
@@ -3279,169 +2870,50 @@ ExecutionSession::IL_emit(MaterializationResponsibility &MR,
return make_error<StringError>("JITDylib " + TargetJD.getName() +
" is defunct",
inconvertibleErrorCode());
+
#ifdef EXPENSIVE_CHECKS
verifySessionState("entering ExecutionSession::IL_emit");
#endif
- // Walk all EDUs:
- // 1. Verifying that dependencies are available (not removed or in the error
- // state.
- // 2. Removing any dependencies that are already Ready.
- // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
- // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
- std::deque<JITDylib::EmissionDepUnit *> Worklist;
- for (auto &[EDU, _] : EDUInfos)
- Worklist.push_back(EDU);
-
- for (auto *EDU : Worklist) {
- auto *EDUInfo = &EDUInfos[EDU];
-
- SmallVector<JITDylib *> DepJDsToRemove;
- for (auto &[DepJD, Deps] : EDU->Dependencies) {
- if (DepJD->State != JITDylib::Open)
- return makeJDClosedError(*EDU, *DepJD);
-
- SymbolNameSet BadDeps;
- SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
- for (auto &Dep : Deps) {
- auto DepEntryItr = DepJD->Symbols.find(SymbolStringPtr(Dep));
-
- // If this dep has been removed or moved to the error state then add it
- // to the bad deps set. We aggregate these bad deps for more
- // comprehensive error messages.
- if (DepEntryItr == DepJD->Symbols.end() ||
- DepEntryItr->second.getFlags().hasError()) {
- BadDeps.insert(SymbolStringPtr(Dep));
- continue;
- }
-
- // If this dep isn't emitted yet then just add it to the NewDeps set to
- // be propagated.
- auto &DepEntry = DepEntryItr->second;
- if (DepEntry.getState() < SymbolState::Emitted) {
- EDUInfo->NewDeps[DepJD].insert(Dep);
- continue;
- }
+ auto ER = G.emit(std::move(SR),
+ [this](JITDylib *JD, NonOwningSymbolStringPtr Name) {
+ return IL_getSymbolState(JD, Name);
+ });
- // This dep has been emitted, so add it to the list to be removed from
- // EDU.
- DepsToRemove.push_back(Dep);
+ EmitQueries EQ;
- // If Dep is Ready then there's nothing further to do.
- if (DepEntry.getState() == SymbolState::Ready) {
- assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
- "Unexpected MaterializationInfo attached to ready symbol");
- continue;
- }
-
- // If we get here then Dep is Emitted. We need to look up its defining
- // EDU and add this EDU to the defining EDU's list of users (this means
- // creating an EDUInfos entry if the defining EDU doesn't have one
- // already).
- assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
- "Expected MaterializationInfo for emitted dependency");
- auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
- assert(DepMI.DefiningEDU &&
- "Emitted symbol does not have a defining EDU");
- assert(DepMI.DependantEDUs.empty() &&
- "Already-emitted symbol has dependant EDUs?");
- auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
- if (!DepEDUInfo.EDU) {
- // No EDUInfo yet -- build initial entry, and reset the EDUInfo
- // pointer, which we will have invalidated.
- EDUInfo = &EDUInfos[EDU];
- DepEDUInfo.EDU = DepMI.DefiningEDU;
- for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
- if (DepDepJD == &TargetJD) {
- for (auto &DepDep : DepDeps)
- if (!MR.getSymbols().count(SymbolStringPtr(DepDep)))
- DepEDUInfo.NewDeps[DepDepJD].insert(DepDep);
- } else
- DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
- }
- }
- DepEDUInfo.IntraEmitUsers.insert(EDU);
- }
-
- // Some dependencies were removed or in an error state -- error out.
- if (!BadDeps.empty())
- return makeUnsatisfiedDepsError(*EDU, *DepJD, std::move(BadDeps));
-
- // Remove the emitted / ready deps from DepJD.
- for (auto &Dep : DepsToRemove)
- Deps.erase(Dep);
-
- // If there are no further deps in DepJD then flag it for removal too.
- if (Deps.empty())
- DepJDsToRemove.push_back(DepJD);
- }
-
- // Remove any JDs whose dependence sets have become empty.
- for (auto &DepJD : DepJDsToRemove) {
- assert(EDU->Dependencies.count(DepJD) &&
- "Trying to remove non-existent dep entries");
- EDU->Dependencies.erase(DepJD);
- }
-
- // Now look for users of this EDU.
- for (auto &[Sym, Flags] : EDU->Symbols) {
- assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
- "Sym not present in symbol table");
- assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
- SymbolState::Resolved ||
- TargetJD.Symbols[SymbolStringPtr(Sym)]
- .getFlags()
- .hasMaterializationSideEffectsOnly()) &&
- "Emitting symbol not in the resolved state");
- assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
- "Symbol is already in an error state");
-
- auto MII = TargetJD.MaterializingInfos.find(SymbolStringPtr(Sym));
- if (MII == TargetJD.MaterializingInfos.end() ||
- MII->second.DependantEDUs.empty())
- continue;
-
- for (auto &DependantEDU : MII->second.DependantEDUs) {
- if (IL_removeEDUDependence(*DependantEDU, TargetJD, Sym, EDUInfos))
- EDUInfo = &EDUInfos[EDU];
- EDUInfo->IntraEmitUsers.insert(DependantEDU);
- }
- MII->second.DependantEDUs.clear();
- }
- }
-
- Worklist.clear();
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
- if (EDUInfo.NewDeps.empty())
- EDUInfo.NewDeps = EDU->Dependencies;
- Worklist.push_back(EDU);
- }
- }
-
- propagateExtraEmitDeps(
- Worklist, EDUInfos,
- [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
- NonOwningSymbolStringPtr Sym) {
- JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(&EDU);
- });
+ // Handle failed queries.
+ for (auto &SN : ER.Failed)
+ IL_collectQueries(
+ EQ.Failed, SN->defs(),
+ [](JITDylib::SymbolTableEntry &E) {
+ E.setFlags(E.getFlags() = JITSymbolFlags::HasError);
+ },
+ [&](AsynchronousSymbolQuery &Q, JITDylib &JD,
+ NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
+ auto &FS = EQ.FailedSymsForQuery[&Q];
+ if (!FS)
+ FS = std::make_shared<SymbolDependenceMap>();
+ (*FS)[&JD].insert(SymbolStringPtr(Name));
+ });
- JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
+ for (auto &FQ : EQ.Failed)
+ FQ->detach();
- // Extract completed queries and lodge not-yet-ready EDUs in the
- // session.
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- if (EDU->Dependencies.empty())
- IL_makeEDUReady(std::move(EDUInfo.EDU), CompletedQueries);
- else
- IL_makeEDUEmitted(std::move(EDUInfo.EDU), CompletedQueries);
- }
+ for (auto &SN : ER.Ready)
+ IL_collectQueries(
+ EQ.Updated, SN->defs(),
+ [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); },
+ [](AsynchronousSymbolQuery &Q, JITDylib &JD,
+ NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) {
+ Q.notifySymbolMetRequiredState(SymbolStringPtr(Name), E.getSymbol());
+ });
#ifdef EXPENSIVE_CHECKS
verifySessionState("exiting ExecutionSession::IL_emit");
#endif
- return std::move(CompletedQueries);
+ return std::move(EQ);
}
Error ExecutionSession::OL_notifyEmitted(
@@ -3471,40 +2943,127 @@ Error ExecutionSession::OL_notifyEmitted(
}
#endif // NDEBUG
- auto EDUInfos = simplifyDepGroups(MR, DepGroups);
+ std::vector<std::unique_ptr<WaitingOnGraph::SuperNode>> SNs;
+ WaitingOnGraph::ContainerElementsMap Residual;
+ {
+ auto &JDResidual = Residual[&MR.getTargetJITDylib()];
+ for (auto &[Name, Flags] : MR.getSymbols())
+ JDResidual.insert(NonOwningSymbolStringPtr(Name));
+
+ for (auto &SDG : DepGroups) {
+ WaitingOnGraph::ContainerElementsMap Defs;
+ assert(!SDG.Symbols.empty());
+ auto &JDDefs = Defs[&MR.getTargetJITDylib()];
+ for (auto &Def : SDG.Symbols) {
+ JDDefs.insert(NonOwningSymbolStringPtr(Def));
+ JDResidual.erase(NonOwningSymbolStringPtr(Def));
+ }
+ WaitingOnGraph::ContainerElementsMap Deps;
+ if (!SDG.Dependencies.empty()) {
+ for (auto &[JD, Syms] : SDG.Dependencies) {
+ auto &JDDeps = Deps[JD];
+ for (auto &Dep : Syms)
+ JDDeps.insert(NonOwningSymbolStringPtr(Dep));
+ }
+ }
+ SNs.push_back(std::make_unique<WaitingOnGraph::SuperNode>(
+ std::move(Defs), std::move(Deps)));
+ }
+ if (!JDResidual.empty())
+ SNs.push_back(std::make_unique<WaitingOnGraph::SuperNode>(
+ std::move(Residual), WaitingOnGraph::ContainerElementsMap()));
+ }
+
+ auto SR = WaitingOnGraph::simplify(std::move(SNs));
LLVM_DEBUG({
dbgs() << " Simplified dependencies:\n";
- for (auto &[EDU, EDUInfo] : EDUInfos) {
- dbgs() << " Symbols: { ";
- for (auto &[Sym, Flags] : EDU->Symbols)
- dbgs() << Sym << " ";
- dbgs() << "}, Dependencies: { ";
- for (auto &[DepJD, Deps] : EDU->Dependencies) {
- dbgs() << "(" << DepJD->getName() << ", { ";
- for (auto &Dep : Deps)
- dbgs() << Dep << " ";
- dbgs() << "}) ";
+ for (auto &SN : SR.superNodes()) {
+
+ auto SortedLibs = [](WaitingOnGraph::ContainerElementsMap &C) {
+ std::vector<JITDylib *> JDs;
+ for (auto &[JD, _] : C)
+ JDs.push_back(JD);
+ llvm::sort(JDs, [](const JITDylib *LHS, const JITDylib *RHS) {
+ return LHS->getName() < RHS->getName();
+ });
+ return JDs;
+ };
+
+ auto SortedNames = [](WaitingOnGraph::ElementSet &Elems) {
+ std::vector<NonOwningSymbolStringPtr> Names(Elems.begin(), Elems.end());
+ llvm::sort(Names, [](const NonOwningSymbolStringPtr &LHS,
+ const NonOwningSymbolStringPtr &RHS) {
+ return *LHS < *RHS;
+ });
+ return Names;
+ };
+
+ dbgs() << " Defs: {";
+ for (auto *JD : SortedLibs(SN->defs())) {
+ dbgs() << " (" << JD->getName() << ", [";
+ for (auto &Sym : SortedNames(SN->defs()[JD]))
+ dbgs() << " " << Sym;
+ dbgs() << " ])";
+ }
+ dbgs() << " }, Deps: {";
+ for (auto *JD : SortedLibs(SN->deps())) {
+ dbgs() << " (" << JD->getName() << ", [";
+ for (auto &Sym : SortedNames(SN->deps()[JD]))
+ dbgs() << " " << Sym;
+ dbgs() << " ])";
}
- dbgs() << "}\n";
+ dbgs() << " }\n";
}
});
-
- auto CompletedQueries =
- runSessionLocked([&]() { return IL_emit(MR, EDUInfos); });
+ auto EmitQueries =
+ runSessionLocked([&]() { return IL_emit(MR, std::move(SR)); });
// On error bail out.
- if (!CompletedQueries)
- return CompletedQueries.takeError();
+ if (!EmitQueries)
+ return EmitQueries.takeError();
- MR.SymbolFlags.clear();
+ // Otherwise notify failed queries, and any updated queries that have been
+ // completed.
- // Otherwise notify all the completed queries.
- for (auto &Q : *CompletedQueries) {
- assert(Q->isComplete() && "Q is not complete");
- Q->handleComplete(*this);
+ // FIXME: Get rid of error return from notifyEmitted.
+ SymbolDependenceMap BadDeps;
+ {
+ for (auto &FQ : EmitQueries->Failed) {
+ FQ->detach();
+ assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) &&
+ "Missing failed symbols for query");
+ auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]);
+ for (auto &[JD, Syms] : *FailedSyms) {
+ auto &BadDepsForJD = BadDeps[JD];
+ for (auto &Sym : Syms)
+ BadDepsForJD.insert(Sym);
+ }
+ FQ->handleFailed(make_error<FailedToMaterialize>(getSymbolStringPool(),
+ std::move(FailedSyms)));
+ }
+ }
+
+ for (auto &UQ : EmitQueries->Updated)
+ if (UQ->isComplete())
+ UQ->handleComplete(*this);
+
+ // If there are any bad dependencies then return an error.
+ if (!BadDeps.empty()) {
+ SymbolNameSet BadNames;
+ // Note: The name set calculated here is bogus: it includes all symbols in
+ // the MR, not just the ones that failed. We want to remove the error
+ // return path from notifyEmitted anyway, so this is just a brief
+ // placeholder to maintain (roughly) the current error behavior.
+ for (auto &[Name, Flags] : MR.getSymbols())
+ BadNames.insert(Name);
+ MR.SymbolFlags.clear();
+ return make_error<UnsatisfiedSymbolDependencies>(
+ getSymbolStringPool(), &MR.getTargetJITDylib(), std::move(BadNames),
+ std::move(BadDeps), "dependencies removed or in error state");
}
+ MR.SymbolFlags.clear();
return Error::success();
}
@@ -3535,158 +3094,48 @@ ExecutionSession::IL_failSymbols(JITDylib &JD,
#endif
JITDylib::AsynchronousSymbolQuerySet FailedQueries;
- auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
- auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
- JITDylib::AsynchronousSymbolQueryList ToDetach;
- for (auto &Q : MI.pendingQueries()) {
- // Add the query to the list to be failed and detach it.
- FailedQueries.insert(Q);
- ToDetach.push_back(Q);
+ auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) {
+ auto I = FailJD->Symbols.find_as(FailSym);
+ assert(I != FailJD->Symbols.end());
+ I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError);
+ auto J = FailJD->MaterializingInfos.find_as(FailSym);
+ if (J != FailJD->MaterializingInfos.end()) {
+ for (auto &Q : J->second.takeAllPendingQueries())
+ FailedQueries.insert(std::move(Q));
+ FailJD->MaterializingInfos.erase(J);
}
- for (auto &Q : ToDetach)
- Q->detach();
- assert(!MI.hasQueriesPending() && "Queries still pending after detach");
};
- for (auto &Name : SymbolsToFail) {
- (*FailedSymbolsMap)[&JD].insert(Name);
-
- // Look up the symbol to fail.
- auto SymI = JD.Symbols.find(Name);
-
- // FIXME: Revisit this. We should be able to assert sequencing between
- // ResourceTracker removal and symbol failure.
- //
- // It's possible that this symbol has already been removed, e.g. if a
- // materialization failure happens concurrently with a ResourceTracker or
- // JITDylib removal. In that case we can safely skip this symbol and
- // continue.
- if (SymI == JD.Symbols.end())
- continue;
- auto &Sym = SymI->second;
-
- // If the symbol is already in the error state then we must have visited
- // it earlier.
- if (Sym.getFlags().hasError()) {
- assert(!JD.MaterializingInfos.count(Name) &&
- "Symbol in error state still has MaterializingInfo");
- continue;
- }
+ auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
- // Move the symbol into the error state.
- Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
-
- // FIXME: Come up with a sane mapping of state to
- // presence-of-MaterializingInfo so that we can assert presence / absence
- // here, rather than testing it.
- auto MII = JD.MaterializingInfos.find(Name);
- if (MII == JD.MaterializingInfos.end())
- continue;
-
- auto &MI = MII->second;
-
- // Collect queries to be failed for this MII.
- ExtractFailedQueries(MI);
-
- if (MI.DefiningEDU) {
- // If there is a DefiningEDU for this symbol then remove this
- // symbol from it.
- assert(MI.DependantEDUs.empty() &&
- "Symbol with DefiningEDU should not have DependantEDUs");
- assert(Sym.getState() >= SymbolState::Emitted &&
- "Symbol has EDU, should have been emitted");
- assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
- "Symbol does not appear in its DefiningEDU");
- MI.DefiningEDU->Symbols.erase(NonOwningSymbolStringPtr(Name));
-
- // Remove this EDU from the dependants lists of its dependencies.
- for (auto &[DepJD, DepSyms] : MI.DefiningEDU->Dependencies) {
- for (auto DepSym : DepSyms) {
- assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
- "DepSym not in DepJD");
- assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
- "DepSym has not MaterializingInfo");
- auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
- assert(SymMI.DependantEDUs.count(MI.DefiningEDU.get()) &&
- "DefiningEDU missing from DependantEDUs list of dependency");
- SymMI.DependantEDUs.erase(MI.DefiningEDU.get());
- }
- }
+ {
+ auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD];
+ for (auto &Sym : SymbolsToFail) {
+ FailedSymsForJD.insert(Sym);
+ Fail(&JD, NonOwningSymbolStringPtr(Sym));
+ }
+ }
- MI.DefiningEDU = nullptr;
- } else {
- // Otherwise if there are any EDUs waiting on this symbol then move
- // those symbols to the error state too, and deregister them from the
- // symbols that they depend on.
- // Note: We use a copy of DependantEDUs here since we'll be removing
- // from the original set as we go.
- for (auto &DependantEDU : MI.DependantEDUs) {
-
- // Remove DependantEDU from all of its users DependantEDUs lists.
- for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
- for (auto DepSym : DepSyms) {
- // Skip self-reference to avoid invalidating the MI.DependantEDUs
- // map. We'll clear this later.
- if (DepJD == &JD && DepSym == Name)
- continue;
- assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
- "DepSym not in DepJD?");
- assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
- "DependantEDU not registered with symbol it depends on");
- auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
- assert(SymMI.DependantEDUs.count(DependantEDU) &&
- "DependantEDU missing from DependantEDUs list");
- SymMI.DependantEDUs.erase(DependantEDU);
- }
- }
+ WaitingOnGraph::ContainerElementsMap ToFail;
+ auto &JDToFail = ToFail[&JD];
+ for (auto &Sym : SymbolsToFail)
+ JDToFail.insert(NonOwningSymbolStringPtr(Sym));
- // Move any symbols defined by DependantEDU into the error state and
- // fail any queries waiting on them.
- auto &DepJD = *DependantEDU->JD;
- auto DepEDUSymbols = std::move(DependantEDU->Symbols);
- for (auto &[DepName, Flags] : DepEDUSymbols) {
- auto DepSymItr = DepJD.Symbols.find(SymbolStringPtr(DepName));
- assert(DepSymItr != DepJD.Symbols.end() &&
- "Symbol not present in table");
- auto &DepSym = DepSymItr->second;
-
- assert(DepSym.getState() >= SymbolState::Emitted &&
- "Symbol has EDU, should have been emitted");
- assert(!DepSym.getFlags().hasError() &&
- "Symbol is already in the error state?");
- DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
- (*FailedSymbolsMap)[&DepJD].insert(SymbolStringPtr(DepName));
-
- // This symbol has a defining EDU so its MaterializingInfo object must
- // exist.
- auto DepMIItr =
- DepJD.MaterializingInfos.find(SymbolStringPtr(DepName));
- assert(DepMIItr != DepJD.MaterializingInfos.end() &&
- "Symbol has defining EDU but not MaterializingInfo");
- auto &DepMI = DepMIItr->second;
- assert(DepMI.DefiningEDU.get() == DependantEDU &&
- "Bad EDU dependence edge");
- assert(DepMI.DependantEDUs.empty() &&
- "Symbol was emitted, should not have any DependantEDUs");
- ExtractFailedQueries(DepMI);
- DepJD.MaterializingInfos.erase(SymbolStringPtr(DepName));
- }
+ auto FailedSNs = G.fail(ToFail);
- DepJD.shrinkMaterializationInfoMemory();
+ for (auto &SN : FailedSNs) {
+ for (auto &[FailJD, Defs] : SN->defs()) {
+ auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD];
+ for (auto &Def : Defs) {
+ FailedSymsForFailJD.insert(SymbolStringPtr(Def));
+ Fail(FailJD, Def);
}
-
- MI.DependantEDUs.clear();
}
-
- assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
- assert(MI.DependantEDUs.empty() &&
- "DependantEDUs should have been removed above");
- assert(!MI.hasQueriesPending() &&
- "Can not delete MaterializingInfo with queries pending");
- JD.MaterializingInfos.erase(Name);
}
- JD.shrinkMaterializationInfoMemory();
+ // Detach all failed queries.
+ for (auto &Q : FailedQueries)
+ Q->detach();
#ifdef EXPENSIVE_CHECKS
verifySessionState("exiting ExecutionSession::IL_failSymbols");
@@ -3721,9 +3170,11 @@ void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
return IL_failSymbols(MR.getTargetJITDylib(), SymbolsToFail);
});
- for (auto &Q : FailedQueries)
+ for (auto &Q : FailedQueries) {
+ Q->detach();
Q->handleFailed(
make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
+ }
}
Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
index dec1df7da2f4a..893523ced8651 100644
--- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
@@ -448,7 +448,7 @@ Error SimpleRemoteEPC::handleHangup(SimpleRemoteEPCArgBytesVector ArgBytes) {
if (const char *ErrMsg = WFR.getOutOfBandError())
return make_error<StringError>(ErrMsg, inconvertibleErrorCode());
- detail::SPSSerializableError Info;
+ orc::shared::detail::SPSSerializableError Info;
SPSInputBuffer IB(WFR.data(), WFR.size());
if (!SPSArgList<SPSError>::deserialize(IB, Info))
return make_error<StringError>("Could not deserialize hangup info",
diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s b/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s
index 83d71cdf6fc83..529395822f5f7 100644
--- a/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s
+++ b/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s
@@ -16,8 +16,7 @@
# CHECK-DAG: Symbols: { _foo }, Dependencies: { (main, { _external_func }) }
# CHECK-DAG: Symbols: { _baz }, Dependencies: { (main, { _foo }) }
# CHECK: Simplified dependencies:
-# CHECK-DAG: Symbols: { _foo }, Dependencies: { (main, { _external_func }) }
-# CHECK-DAG: Symbols: { _baz }, Dependencies: { (main, { _external_func }) }
+# CHECK-DAG: Defs: { (main, [ _baz _foo ]) }, Deps: { (main, [ _external_func ]) }
.section __TEXT,__text,regular,pure_instructions
diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
index a2bbb10039c9a..7e3ebc88cea63 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
@@ -42,6 +42,7 @@ add_llvm_unittest(OrcJITTests
SymbolStringPoolTest.cpp
TaskDispatchTest.cpp
ThreadSafeModuleTest.cpp
+ WaitingOnGraphTest.cpp
WrapperFunctionUtilsTest.cpp
JITLinkRedirectionManagerTest.cpp
ReOptimizeLayerTest.cpp
diff --git a/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp
new file mode 100644
index 0000000000000..b988a78a3783a
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp
@@ -0,0 +1,553 @@
+//===--------- WaitingOnGraphTest.cpp - Test WaitingOnGraph APIs ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/WaitingOnGraph.h"
+#include "gtest/gtest.h"
+
+namespace llvm::orc::detail {
+
+class WaitingOnGraphTest : public testing::Test {
+public:
+ using TestGraph = WaitingOnGraph<uintptr_t, uintptr_t>;
+
+protected:
+ using SuperNode = TestGraph::SuperNode;
+ using SuperNodeBuilder = TestGraph::SuperNodeBuilder;
+ using ContainerElementsMap = TestGraph::ContainerElementsMap;
+ using ElemToSuperNodeMap = TestGraph::ElemToSuperNodeMap;
+ using SimplifyResult = TestGraph::SimplifyResult;
+ using EmitResult = TestGraph::EmitResult;
+
+ static const ContainerElementsMap &getDefs(SuperNode &SN) { return SN.Defs; }
+
+ static const ContainerElementsMap &getDeps(SuperNode &SN) { return SN.Deps; }
+
+ static std::vector<std::unique_ptr<SuperNode>> &getSNs(SimplifyResult &SR) {
+ return SR.SNs;
+ }
+
+ static ElemToSuperNodeMap &getElemToSN(SimplifyResult &SR) {
+ return SR.ElemToSN;
+ }
+
+ static std::vector<std::unique_ptr<SuperNode>> &getPendingSNs(TestGraph &G) {
+ return G.PendingSNs;
+ }
+
+ static ContainerElementsMap merge(ContainerElementsMap M1,
+ const ContainerElementsMap &M2) {
+ ContainerElementsMap Result = std::move(M1);
+ for (auto &[Container, Elems] : M2)
+ Result[Container].insert(Elems.begin(), Elems.end());
+ return Result;
+ }
+
+ ContainerElementsMap
+ collapseDefs(std::vector<std::unique_ptr<SuperNode>> &SNs,
+ bool DepsMustMatch = true) {
+ if (SNs.empty())
+ return ContainerElementsMap();
+
+ ContainerElementsMap Result = SNs[0]->defs();
+ const ContainerElementsMap &Deps = SNs[0]->deps();
+
+ for (size_t I = 1; I != SNs.size(); ++I) {
+ assert(!DepsMustMatch || SNs[I]->deps() == Deps);
+ Result = merge(std::move(Result), SNs[I]->defs());
+ }
+
+ return Result;
+ }
+
+ EmitResult integrate(EmitResult ER) {
+ for (auto &SN : ER.Ready)
+ for (auto &[Container, Elems] : SN->defs())
+ Ready[Container].insert(Elems.begin(), Elems.end());
+ for (auto &SN : ER.Failed)
+ for (auto &[Container, Elems] : SN->defs())
+ Failed[Container].insert(Elems.begin(), Elems.end());
+ return ER;
+ }
+
+ EmitResult emit(SimplifyResult SR) {
+ return integrate(G.emit(std::move(SR), GetExternalState));
+ }
+
+ TestGraph G;
+ ContainerElementsMap Ready;
+ ContainerElementsMap Failed;
+
+ class ExternalStateGetter {
+ public:
+ ExternalStateGetter(WaitingOnGraphTest &T) : T(T) {}
+ TestGraph::ExternalState operator()(TestGraph::ContainerId C,
+ TestGraph::ElementId E) {
+ {
+ auto I = T.Failed.find(C);
+ if (I != T.Failed.end())
+ if (I->second.count(E))
+ return TestGraph::ExternalState::Failed;
+ }
+
+ {
+ auto I = T.Ready.find(C);
+ if (I != T.Ready.end())
+ if (I->second.count(E))
+ return TestGraph::ExternalState::Ready;
+ }
+
+ return TestGraph::ExternalState::None;
+ }
+
+ private:
+ WaitingOnGraphTest &T;
+ };
+
+ ExternalStateGetter GetExternalState{*this};
+};
+
+} // namespace llvm::orc::detail
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::orc::detail;
+
+TEST_F(WaitingOnGraphTest, ConstructAndDestroyEmpty) {
+ // Nothing to do here -- we're just testing construction and destruction
+ // of the WaitingOnGraphTest::G member.
+}
+
+TEST_F(WaitingOnGraphTest, Build_TrivialSingleSuperNode) {
+ // Add one set of trivial defs and empty deps to the builder, make sure that
+ // they're passed through to the resulting super-node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0}}});
+ ContainerElementsMap Deps;
+ B.add(Defs, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), Defs);
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_EmptyDefs) {
+ // Adding empty def sets is ok, but should not result in creation of a
+ // SuperNode.
+ SuperNodeBuilder B;
+ ContainerElementsMap Empty;
+ B.add(Empty, Empty);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_TRUE(SNs.empty());
+}
+
+TEST_F(WaitingOnGraphTest, Build_NonTrivialSingleSuperNode) {
+ // Add one non-trivwial set of defs and deps. Make sure that they're passed
+ // through to the resulting super-node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0, 1, 2}}});
+ ContainerElementsMap Deps({{1, {3, 4, 5}}});
+ B.add(Defs, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), Defs);
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_CoalesceEmptyDeps) {
+ // Add two trivial defs both with empty deps to the builder. Check that
+ // they're coalesced into a single super-node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs1({{0, {0}}});
+ ContainerElementsMap Defs2({{0, {1}}});
+ ContainerElementsMap Deps;
+ B.add(Defs1, Deps);
+ B.add(Defs2, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), merge(Defs1, Defs2));
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_CoalesceNonEmptyDeps) {
+ // Add two sets trivial of trivial defs with empty deps to the builder. Check
+ // that the two coalesce into a single super node.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs1({{0, {0}}});
+ ContainerElementsMap Defs2({{0, {1}}});
+ ContainerElementsMap Deps({{1, {1}}});
+ B.add(Defs1, Deps);
+ B.add(Defs2, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), merge(Defs1, Defs2));
+ EXPECT_EQ(getDeps(*SNs[0]), Deps);
+}
+
+TEST_F(WaitingOnGraphTest, Build_CoalesceInterleaved) {
+ // Add multiple sets of defs, some with the same dep sets. Check that nodes
+ // are still coalesced as expected.
+ SuperNodeBuilder B;
+
+ ContainerElementsMap DefsA1({{0, {0}}});
+ ContainerElementsMap DefsA2({{0, {1}}});
+ ContainerElementsMap DefsB1({{1, {0}}});
+ ContainerElementsMap DefsB2({{1, {1}}});
+ ContainerElementsMap DepsA({{2, {0}}, {3, {0}}});
+ ContainerElementsMap DepsB({{4, {0}}, {5, {0}}});
+ B.add(DefsA1, DepsA);
+ B.add(DefsB1, DepsB);
+ B.add(DefsA2, DepsA);
+ B.add(DefsB2, DepsB);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 2U);
+ EXPECT_EQ(getDefs(*SNs[0]), merge(DefsA1, DefsA2));
+ EXPECT_EQ(getDeps(*SNs[0]), DepsA);
+ EXPECT_EQ(getDefs(*SNs[1]), merge(DefsB1, DefsB2));
+ EXPECT_EQ(getDeps(*SNs[1]), DepsB);
+}
+
+TEST_F(WaitingOnGraphTest, Build_SelfDepRemoval) {
+ // Add multiple sets of defs, some with the same dep sets. Check that nodes
+ // are still coalesced as expected.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0, 1}}});
+ ContainerElementsMap Deps({{0, {1}}});
+ ContainerElementsMap Empty;
+ B.add(Defs, Deps);
+ auto SNs = B.takeSuperNodes();
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs[0]), Defs);
+ EXPECT_EQ(getDeps(*SNs[0]), Empty);
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_EmptySimplification) {
+ auto SR = TestGraph::simplify({});
+ auto &SNs = getSNs(SR);
+ EXPECT_EQ(SNs.size(), 0U);
+ EXPECT_EQ(getElemToSN(SR), ElemToSuperNodeMap());
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_TrivialSingleSuperNode) {
+ // Test trivial call to simplify.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0}}});
+ ContainerElementsMap Deps({{0, {0}}});
+ B.add(Defs, Deps);
+ auto SR = TestGraph::simplify(B.takeSuperNodes());
+ ContainerElementsMap Empty;
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs.at(0)), Defs);
+ EXPECT_EQ(getDeps(*SNs.at(0)), Empty);
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ ExpectedElemToSNs[0][0] = SNs[0].get();
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_SimplifySingleContainerSimpleCycle) {
+ // Test trivial simplification call with two nodes and one internal
+ // dependence cycle within a single container:
+ // N0: (0, 0) -> (0, 1)
+ // N1: (0, 1) -> (0, 0)
+ // We expect intra-simplify cycle elimination to clear both dependence sets,
+ // and coalescing to join them into one supernode covering both defs.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {0}}});
+ B.add(Defs1, Deps1);
+ auto SR = TestGraph::simplify(B.takeSuperNodes());
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ ContainerElementsMap Empty;
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs.at(0)), merge(Defs0, Defs1));
+ EXPECT_EQ(getDeps(*SNs.at(0)), Empty);
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ ExpectedElemToSNs[0][0] = SNs[0].get();
+ ExpectedElemToSNs[0][1] = SNs[0].get();
+
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest,
+ Simplification_SimplifySingleContainerNElementCycle) {
+ // Test trivial simplification call with M nodes and one internal
+ // dependence cycle within a single container:
+ // N0: (0, 0) -> (0, 1)
+ // N1: (0, 1) -> (0, 2)
+ // ...
+ // NM: (0, M) -> (0, 0)
+ // We expect intra-simplify cycle elimination to clear all dependence sets,
+ // and coalescing to join them into one supernode covering all defs.
+ SuperNodeBuilder B;
+ constexpr size_t M = 10;
+ for (size_t I = 0; I != M; ++I) {
+ ContainerElementsMap Defs({{0, {I}}});
+ ContainerElementsMap Deps({{0, {(I + 1) % M}}});
+ B.add(Defs, Deps);
+ }
+ auto InitSNs = B.takeSuperNodes();
+ EXPECT_EQ(InitSNs.size(), M);
+
+ auto SR = TestGraph::simplify(std::move(InitSNs));
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ ContainerElementsMap ExpectedDefs;
+ for (size_t I = 0; I != M; ++I)
+ ExpectedDefs[0].insert(I);
+ ContainerElementsMap Empty;
+ EXPECT_EQ(SNs.size(), 1U);
+ EXPECT_EQ(getDefs(*SNs.at(0)), ExpectedDefs);
+ EXPECT_EQ(getDeps(*SNs.at(0)), Empty);
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ for (size_t I = 0; I != M; ++I)
+ ExpectedElemToSNs[0][I] = SNs[0].get();
+
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest, Simplification_SimplifyIntraSimplifyPropagateDeps) {
+ // Test trivial simplification call with two nodes and one internal
+ // dependence cycle within a single container:
+ // N0: (0, 0) -> (0, {1, 2})
+ // N1: (0, 1) -> (0, {3})
+ // We expect intra-simplify cycle elimination to replace the dependence of
+ // (0, 0) on (0, 1) with a dependence on (0, 3) instead.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1, 2}}});
+ B.add(Defs0, Deps0);
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {3}}});
+ B.add(Defs1, Deps1);
+ auto SR = TestGraph::simplify(B.takeSuperNodes());
+
+ // Check SNs.
+ auto &SNs = getSNs(SR);
+ EXPECT_EQ(SNs.size(), 2U);
+
+ // ContainerElemenstMap ExpectedDefs0({{0, {0}}});
+ // ContainerElemenstMap ExpectedDeps0({{0, {1, 3}}});
+ EXPECT_EQ(getDefs(*SNs.at(0)), ContainerElementsMap({{0, {0}}}));
+ EXPECT_EQ(getDeps(*SNs.at(0)), ContainerElementsMap({{0, {2, 3}}}));
+
+ EXPECT_EQ(getDefs(*SNs.at(1)), ContainerElementsMap({{0, {1}}}));
+ EXPECT_EQ(getDeps(*SNs.at(1)), ContainerElementsMap({{0, {3}}}));
+
+ // Check ElemToSNs.
+ ElemToSuperNodeMap ExpectedElemToSNs;
+ ExpectedElemToSNs[0][0] = SNs[0].get();
+ ExpectedElemToSNs[0][1] = SNs[1].get();
+
+ EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_EmptyEmit) {
+ // Check that empty emits work as expected.
+ auto ER = G.emit(TestGraph::simplify({}), GetExternalState);
+
+ EXPECT_EQ(ER.Ready.size(), 0U);
+ EXPECT_EQ(ER.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_TrivialSingleNode) {
+ // Check that emitting a single node behaves as expected.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {0}}});
+ B.add(Defs, ContainerElementsMap());
+ auto ER = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER.Ready), Defs);
+ EXPECT_EQ(ER.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_TrivialSequence) {
+ // Perform a sequence of two emits where the second emit depends on the
+ // first. Check that nodes become ready after each emit.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Empty;
+ B.add(Defs0, Empty);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER0.Ready), Defs0);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {0}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER1.Ready), Defs1);
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_TrivialReverseSequence) {
+ // Perform a sequence of two emits where the first emit depends on the
+ // second. Check that both nodes become ready after the second emit.
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Empty;
+ B.add(Defs1, Empty);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER1.Ready), merge(Defs0, Defs1));
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_Coalescing) {
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{1, {0}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{1, {0}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER1.Ready.size(), 0U);
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+
+ // Check that after emitting two nodes with the same dep set we have only one
+ // pending supernode whose defs are the union of the defs in the two emits.
+ auto &PendingSNs = getPendingSNs(G);
+ EXPECT_EQ(PendingSNs.size(), 1U);
+ EXPECT_EQ(getDefs(*PendingSNs.at(0)), merge(Defs0, Defs1));
+
+ ContainerElementsMap Defs2({{1, {0}}});
+ ContainerElementsMap Empty;
+ B.add(Defs2, Empty);
+ auto ER2 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(collapseDefs(ER2.Ready), merge(merge(Defs0, Defs1), Defs2));
+ EXPECT_EQ(ER2.Failed.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Emit_ZigZag) {
+ // Perform a sequence of four emits, where the first three contain a zig-zag
+ // pattern:
+ // 1. (0, 0) -> (0, 1)
+ // 2. (0, 2) -> (0, 3)
+ // ^ -- At this point we expect two pending supernodes.
+ // 3. (0, 1) -> (0, 2)
+ // ^ -- Resolution of (0, 1) should cause all three emitted nodes to coalsce
+ // into one supernode defining (0, {1, 2, 3}).
+ // 4. (0, 3)
+ // ^ -- Should cause all four nodes to become ready.
+
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ ContainerElementsMap Defs1({{0, {2}}});
+ ContainerElementsMap Deps1({{0, {3}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER1.Ready.size(), 0U);
+ EXPECT_EQ(ER1.Failed.size(), 0U);
+
+ // Check that after emitting two nodes with the same dep set we have only one
+ // pending supernode whose defs are the union of the defs in the two emits.
+ auto &PendingSNs = getPendingSNs(G);
+ EXPECT_EQ(PendingSNs.size(), 2U);
+ EXPECT_EQ(getDefs(*PendingSNs.at(0)), Defs0);
+ EXPECT_EQ(getDeps(*PendingSNs.at(0)), Deps0);
+ EXPECT_EQ(getDefs(*PendingSNs.at(1)), Defs1);
+ EXPECT_EQ(getDeps(*PendingSNs.at(1)), Deps1);
+
+ ContainerElementsMap Defs2({{0, {1}}});
+ ContainerElementsMap Deps2({{0, {2}}});
+ B.add(Defs2, Deps2);
+ auto ER2 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER2.Ready.size(), 0U);
+ EXPECT_EQ(ER2.Failed.size(), 0U);
+
+ // Check that after emitting the third node we've coalesced all three.
+ EXPECT_EQ(PendingSNs.size(), 1U);
+ EXPECT_EQ(getDefs(*PendingSNs.at(0)), merge(merge(Defs0, Defs1), Defs2));
+ EXPECT_EQ(getDeps(*PendingSNs.at(0)), Deps1);
+
+ ContainerElementsMap Defs3({{0, {3}}});
+ ContainerElementsMap Empty;
+ B.add(Defs3, Empty);
+ auto ER3 = emit(TestGraph::simplify(B.takeSuperNodes()));
+
+ EXPECT_EQ(collapseDefs(ER3.Ready),
+ merge(merge(merge(Defs0, Defs1), Defs2), Defs3));
+ EXPECT_EQ(ER2.Failed.size(), 0U);
+ EXPECT_TRUE(PendingSNs.empty());
+}
+
+TEST_F(WaitingOnGraphTest, Fail_Empty) {
+ // Check that failing an empty set is a no-op.
+ auto FR = G.fail(ContainerElementsMap());
+ EXPECT_EQ(FR.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Fail_Single) {
+ // Check that failing a set with no existing dependencies works.
+ auto FR = G.fail({{0, {0}}});
+ EXPECT_EQ(FR.size(), 0U);
+}
+
+TEST_F(WaitingOnGraphTest, Fail_EmitDependenceOnFailure) {
+ // Check that emitted nodes that directly depend on failed nodes also fail.
+ Failed = {{0, {0}}};
+
+ SuperNodeBuilder B;
+ ContainerElementsMap Defs({{0, {1}}});
+ ContainerElementsMap Deps({{0, {0}}});
+ B.add(Defs, Deps);
+ auto ER = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER.Ready.size(), 0U);
+ EXPECT_EQ(collapseDefs(ER.Failed, false), Defs);
+}
+
+TEST_F(WaitingOnGraphTest, Fail_ZigZag) {
+ // Check that if an emit introduces a transitive dependence of a failed
+ // node, then all nodes that depend on the failed node are also failed.
+ SuperNodeBuilder B;
+
+ ContainerElementsMap Defs0({{0, {0}}});
+ ContainerElementsMap Deps0({{0, {1}}});
+ B.add(Defs0, Deps0);
+ auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER0.Ready.size(), 0U);
+ EXPECT_EQ(ER0.Failed.size(), 0U);
+
+ Failed = {{0, {2}}};
+
+ ContainerElementsMap Defs1({{0, {1}}});
+ ContainerElementsMap Deps1({{0, {2}}});
+ B.add(Defs1, Deps1);
+ auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes()));
+ EXPECT_EQ(ER1.Ready.size(), 0U);
+ EXPECT_EQ(collapseDefs(ER1.Failed, false), merge(Defs0, Defs1));
+}
More information about the llvm-commits
mailing list