[llvm] f597ce0 - [ORC][Runtime] Enhancing ELF Platform with Push-Request Model for Initializers (#102846)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 28 00:33:24 PDT 2024
Author: SahilPatidar
Date: 2024-09-28T17:33:19+10:00
New Revision: f597ce03a6b2b7268e6a5c11cf7478c87b65b2bc
URL: https://github.com/llvm/llvm-project/commit/f597ce03a6b2b7268e6a5c11cf7478c87b65b2bc
DIFF: https://github.com/llvm/llvm-project/commit/f597ce03a6b2b7268e6a5c11cf7478c87b65b2bc.diff
LOG: [ORC][Runtime] Enhancing ELF Platform with Push-Request Model for Initializers (#102846)
With the help of @lhames, this PR introduces a push-request model to manage
initializers in the runtime state for each `JITDylib`, similar to how `MachO`
and `COFF` handle it. Previously, the ELF runtime lacked the ability to
register, deregister, or retain initializers, causing issues when re-running
`dlopen`. Initializers were erased after `rt_getInitializers` was used.
To address this, we introduce `__orc_rt_elfnix_register_init_sections` and
`__orc_rt_elfnix_register_jitdylib` (and corresponding deregister functions).
This model allows the runtime to request the push of initializers for the
`JITDylib`, with the platform handling this via a record method. Additionally,
we leverage `MachO`'s `RecordSectionsTracker` to store initializers, ensuring
only newly updated initializers are executed using `processNewSections`,
improving the runtime's efficiency and reliability.
Added:
compiler-rt/lib/orc/sections_tracker.h
Modified:
compiler-rt/lib/orc/elfnix_platform.cpp
compiler-rt/lib/orc/elfnix_platform.h
compiler-rt/lib/orc/macho_platform.cpp
llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
Removed:
################################################################################
diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp
index bd76e3ed0ece7c..456d50dd90212a 100644
--- a/compiler-rt/lib/orc/elfnix_platform.cpp
+++ b/compiler-rt/lib/orc/elfnix_platform.cpp
@@ -15,6 +15,7 @@
#include "compiler.h"
#include "error.h"
#include "jit_dispatch.h"
+#include "sections_tracker.h"
#include "wrapper_function_utils.h"
#include <algorithm>
@@ -29,8 +30,7 @@ using namespace orc_rt;
using namespace orc_rt::elfnix;
// Declare function tags for functions in the JIT process.
-ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
-ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
// eh-frame registration functions, made available via aliases
@@ -57,21 +57,6 @@ Error validatePointerSectionExtent(const char *SectionName,
return Error::success();
}
-Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
- const ELFNixJITDylibInitializers &MOJDIs) {
-
- for (const auto &ModInits : InitArraySections) {
- if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
- return Err;
-
- using InitFunc = void (*)();
- for (auto *Init : ModInits.toSpan<InitFunc>())
- (*Init)();
- }
-
- return Error::success();
-}
-
struct TLSInfoEntry {
unsigned long Key = 0;
unsigned long DataAddress = 0;
@@ -92,10 +77,18 @@ class ELFNixPlatformRuntimeState {
using AtExitsVector = std::vector<AtExitEntry>;
struct PerJITDylibState {
+ std::string Name;
void *Header = nullptr;
size_t RefCount = 0;
+ size_t LinkedAgainstRefCount = 0;
bool AllowReinitialization = false;
AtExitsVector AtExits;
+ std::vector<PerJITDylibState *> Deps;
+ RecordSectionsTracker<void (*)()> RecordedInits;
+
+ bool referenced() const {
+ return LinkedAgainstRefCount != 0 || RefCount != 0;
+ }
};
public:
@@ -113,6 +106,12 @@ class ELFNixPlatformRuntimeState {
ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
+ Error registerJITDylib(std::string &Name, void *Handle);
+ Error deregisterJITDylib(void *Handle);
+ Error registerInits(ExecutorAddr HeaderAddr,
+ std::vector<ExecutorAddrRange> Inits);
+ Error deregisterInits(ExecutorAddr HeaderAddr,
+ std::vector<ExecutorAddrRange> Inits);
Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
const char *dlerror();
@@ -122,6 +121,8 @@ class ELFNixPlatformRuntimeState {
int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
void runAtExits(void *DSOHandle);
+ void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock,
+ PerJITDylibState &JDS);
/// Returns the base address of the section containing ThreadData.
Expected<std::pair<const char *, size_t>>
@@ -132,18 +133,23 @@ class ELFNixPlatformRuntimeState {
private:
PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
PerJITDylibState *getJITDylibStateByName(std::string_view Path);
- PerJITDylibState &
- getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
Error registerThreadDataSection(span<const char> ThreadDataSection);
Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
std::string_view Symbol);
- Expected<ELFNixJITDylibInitializerSequence>
- getJITDylibInitializersByName(std::string_view Path);
- Expected<void *> dlopenInitialize(std::string_view Path, int Mode);
- Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
+ Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock,
+ PerJITDylibState &JDS);
+ Expected<void *> dlopenImpl(std::string_view Path, int Mode);
+ Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
+ PerJITDylibState &JDS);
+ Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
+ PerJITDylibState &JDS,
+ ELFNixJITDylibDepInfoMap &DepInfo);
+ Error dlcloseImpl(void *DSOHandle);
+ Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
+ PerJITDylibState &JDS);
static ELFNixPlatformRuntimeState *MOPS;
@@ -215,31 +221,110 @@ Error ELFNixPlatformRuntimeState::deregisterObjectSections(
return Error::success();
}
-const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name,
+ void *Handle) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
-void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
+ if (JDStates.count(Handle)) {
+ std::ostringstream ErrStream;
+ ErrStream << "Duplicate JITDylib registration for header " << Handle
+ << " (name = " << Name << ")";
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ if (JDNameToHeader.count(Name)) {
+ std::ostringstream ErrStream;
+ ErrStream << "Duplicate JITDylib registration for header " << Handle
+ << " (header = " << Handle << ")";
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ auto &JD = JDStates[Handle];
+ JD.Header = Handle;
+ JD.Name = std::move(Name);
+ JDNameToHeader[JD.Name] = Handle;
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) {
std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
- // Use fast path if all JITDylibs are already loaded and don't require
- // re-running initializers.
- if (auto *JDS = getJITDylibStateByName(Path)) {
- if (!JDS->AllowReinitialization) {
- ++JDS->RefCount;
- return JDS->Header;
- }
+ auto I = JDStates.find(Handle);
+ if (I == JDStates.end()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Attempted to deregister unrecognized header " << Handle;
+ return make_error<StringError>(ErrStream.str());
}
- auto H = dlopenInitialize(Path, Mode);
- if (!H) {
+ auto J = JDNameToHeader.find(
+ std::string(I->second.Name.data(), I->second.Name.size()));
+ assert(J != JDNameToHeader.end() &&
+ "Missing JDNameToHeader entry for JITDylib");
+ JDNameToHeader.erase(J);
+ JDStates.erase(I);
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::registerInits(
+ ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ PerJITDylibState *JDS =
+ getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
+
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Could not register object platform sections for "
+ "unrecognized header "
+ << HeaderAddr.toPtr<void *>();
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ for (auto &I : Inits) {
+ JDS->RecordedInits.add(I.toSpan<void (*)()>());
+ }
+
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::deregisterInits(
+ ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
+
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ PerJITDylibState *JDS =
+ getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
+
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Could not register object platform sections for unrecognized "
+ "header "
+ << HeaderAddr.toPtr<void *>();
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ for (auto &I : Inits) {
+ JDS->RecordedInits.removeIfPresent(I);
+ }
+
+ return Error::success();
+}
+
+const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+
+void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
+ if (auto H = dlopenImpl(Path, Mode))
+ return *H;
+ else {
+ // FIXME: Make dlerror thread safe.
DLFcnError = toString(H.takeError());
return nullptr;
}
-
- return *H;
}
int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
- runAtExits(DSOHandle);
+ if (auto Err = dlcloseImpl(DSOHandle)) {
+ DLFcnError = toString(std::move(Err));
+ return -1;
+ }
return 0;
}
@@ -265,15 +350,17 @@ int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
}
void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
- // FIXME: Should atexits be allowed to run concurrently with access to
- // JDState?
- AtExitsVector V;
- {
- std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
- auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
- assert(JDS && "JITDlybi state not initialized");
- std::swap(V, JDS->AtExits);
- }
+ std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
+ PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+
+ if (JDS)
+ runAtExits(Lock, *JDS);
+}
+
+void ELFNixPlatformRuntimeState::runAtExits(
+ std::unique_lock<std::recursive_mutex> &JDStateLock,
+ PerJITDylibState &JDS) {
+ AtExitsVector V = std::move(JDS.AtExits);
while (!V.empty()) {
auto &AE = V.back();
@@ -300,6 +387,7 @@ ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
auto I = JDStates.find(DSOHandle);
if (I == JDStates.end())
return nullptr;
+
return &I->second;
}
@@ -316,24 +404,6 @@ ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
return &J->second;
}
-ELFNixPlatformRuntimeState::PerJITDylibState &
-ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
- ELFNixJITDylibInitializers &MOJDIs) {
- void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
-
- auto &JDS = JDStates[Header];
-
- // If this entry hasn't been created yet.
- if (!JDS.Header) {
- assert(!JDNameToHeader.count(MOJDIs.Name) &&
- "JITDylib has header map entry but no name map entry");
- JDNameToHeader[MOJDIs.Name] = Header;
- JDS.Header = Header;
- }
-
- return JDS;
-}
-
Error ELFNixPlatformRuntimeState::registerThreadDataSection(
span<const char> ThreadDataSection) {
std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
@@ -360,74 +430,142 @@ ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
return Result;
}
-Expected<ELFNixJITDylibInitializerSequence>
-ELFNixPlatformRuntimeState::getJITDylibInitializersByName(
- std::string_view Path) {
- Expected<ELFNixJITDylibInitializerSequence> Result(
- (ELFNixJITDylibInitializerSequence()));
- std::string PathStr(Path.data(), Path.size());
- if (auto Err =
- WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
- SPSString)>::
- call(JITDispatch(&__orc_rt_elfnix_get_initializers_tag), Result,
- Path))
- return std::move(Err);
- return Result;
+Error ELFNixPlatformRuntimeState::runInits(
+ std::unique_lock<std::recursive_mutex> &JDStatesLock,
+ PerJITDylibState &JDS) {
+ std::vector<span<void (*)()>> InitSections;
+ InitSections.reserve(JDS.RecordedInits.numNewSections());
+
+ JDS.RecordedInits.processNewSections(
+ [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
+
+ JDStatesLock.unlock();
+ for (auto Sec : InitSections)
+ for (auto *Init : Sec)
+ Init();
+
+ JDStatesLock.lock();
+
+ return Error::success();
}
-Expected<void *>
-ELFNixPlatformRuntimeState::dlopenInitialize(std::string_view Path, int Mode) {
- // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
- // reinitialization. We need to call in to the JIT to see if there's any new
- // work pending.
- auto InitSeq = getJITDylibInitializersByName(Path);
- if (!InitSeq)
- return InitSeq.takeError();
-
- // Init sequences should be non-empty.
- if (InitSeq->empty())
- return make_error<StringError>(
- "__orc_rt_elfnix_get_initializers returned an "
- "empty init sequence");
-
- // Otherwise register and run initializers for each JITDylib.
- for (auto &MOJDIs : *InitSeq)
- if (auto Err = initializeJITDylib(MOJDIs))
- return std::move(Err);
-
- // Return the header for the last item in the list.
- auto *JDS = getJITDylibStateByHeaderAddr(
- InitSeq->back().DSOHandleAddress.toPtr<void *>());
- assert(JDS && "Missing state entry for JD");
+Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path,
+ int Mode) {
+ std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
+ PerJITDylibState *JDS = getJITDylibStateByName(Path);
+
+ if (!JDS)
+ return make_error<StringError>("No registered JTIDylib for path " +
+ std::string(Path.data(), Path.size()));
+
+ if (auto Err = dlopenFull(Lock, *JDS))
+ return std::move(Err);
+
+ ++JDS->RefCount;
+
return JDS->Header;
}
-long getPriority(const std::string &name) {
- auto pos = name.find_last_not_of("0123456789");
- if (pos == name.size() - 1)
- return 65535;
- else
- return std::strtol(name.c_str() + pos + 1, nullptr, 10);
+Error ELFNixPlatformRuntimeState::dlopenFull(
+ std::unique_lock<std::recursive_mutex> &JDStateLock,
+ PerJITDylibState &JDS) {
+ Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
+ JDStateLock.unlock();
+ if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
+ SPSExecutorAddr)>::
+ call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
+ ExecutorAddr::fromPtr(JDS.Header)))
+ return Err;
+ JDStateLock.lock();
+
+ if (!DepInfo)
+ return DepInfo.takeError();
+
+ if (auto Err = dlopenInitialize(JDStateLock, JDS, *DepInfo))
+ return Err;
+
+ if (!DepInfo->empty()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Encountered unrecognized dep-info key headers "
+ "while processing dlopen of "
+ << JDS.Name;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ return Error::success();
}
-Error ELFNixPlatformRuntimeState::initializeJITDylib(
- ELFNixJITDylibInitializers &MOJDIs) {
+Error ELFNixPlatformRuntimeState::dlopenInitialize(
+ std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS,
+ ELFNixJITDylibDepInfoMap &DepInfo) {
+
+ auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
+ if (I == DepInfo.end())
+ return Error::success();
- auto &JDS = getOrCreateJITDylibState(MOJDIs);
- ++JDS.RefCount;
+ auto Deps = std::move(I->second);
+ DepInfo.erase(I);
- using SectionList = std::vector<ExecutorAddrRange>;
- std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(),
- [](const std::pair<std::string, SectionList> &LHS,
- const std::pair<std::string, SectionList> &RHS) -> bool {
- return getPriority(LHS.first) < getPriority(RHS.first);
- });
- for (auto &Entry : MOJDIs.InitSections)
- if (auto Err = runInitArray(Entry.second, MOJDIs))
+ std::vector<PerJITDylibState *> OldDeps;
+ std::swap(JDS.Deps, OldDeps);
+ JDS.Deps.reserve(Deps.size());
+ for (auto H : Deps) {
+ PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(H.toPtr<void *>());
+ if (!DepJDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Encountered unrecognized dep header " << H.toPtr<void *>()
+ << " while initializing " << JDS.Name;
+ return make_error<StringError>(ErrStream.str());
+ }
+ ++DepJDS->LinkedAgainstRefCount;
+ if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
return Err;
+ }
+
+ if (auto Err = runInits(JDStatesLock, JDS))
+ return Err;
+
+ for (auto *DepJDS : OldDeps) {
+ --DepJDS->LinkedAgainstRefCount;
+ if (!DepJDS->referenced())
+ if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
+ return Err;
+ }
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
+
+ std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
+ PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "No registered JITDylib for " << DSOHandle;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ --JDS->RefCount;
+
+ if (!JDS->referenced())
+ return dlcloseInitialize(Lock, *JDS);
+
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::dlcloseInitialize(
+ std::unique_lock<std::recursive_mutex> &JDStatesLock,
+ PerJITDylibState &JDS) {
+ runAtExits(JDStatesLock, JDS);
+ JDS.RecordedInits.reset();
+ for (auto *DepJDS : JDS.Deps)
+ if (!JDS.referenced())
+ if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
+ return Err;
return Error::success();
}
+
class ELFNixPlatformRuntimeTLVManager {
public:
void *getInstance(const char *ThreadData);
@@ -473,19 +611,73 @@ void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
- return WrapperFunction<void(uint64_t)>::handle(
+ return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
ArgData, ArgSize,
- [](uint64_t &DSOHandle) {
+ [](ExecutorAddr DSOHandle) {
ELFNixPlatformRuntimeState::initialize(
- reinterpret_cast<void *>(DSOHandle));
+ DSOHandle.toPtr<void *>());
+ return Error::success();
})
.release();
}
ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
- ELFNixPlatformRuntimeState::destroy();
- return WrapperFunctionResult().release();
+ return WrapperFunction<SPSError()>::handle(
+ ArgData, ArgSize,
+ []() {
+ ELFNixPlatformRuntimeState::destroy();
+ return Error::success();
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
+ ArgData, ArgSize,
+ [](std::string &JDName, ExecutorAddr HeaderAddr) {
+ return ELFNixPlatformRuntimeState::get().registerJITDylib(
+ JDName, HeaderAddr.toPtr<void *>());
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
+ ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr) {
+ return ELFNixPlatformRuntimeState::get().deregisterJITDylib(
+ HeaderAddr.toPtr<void *>());
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr,
+ SPSSequence<SPSExecutorAddrRange>)>::
+ handle(ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr,
+ std::vector<ExecutorAddrRange> &Inits) {
+ return ELFNixPlatformRuntimeState::get().registerInits(
+ HeaderAddr, std::move(Inits));
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr,
+ SPSSequence<SPSExecutorAddrRange>)>::
+ handle(ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr,
+ std::vector<ExecutorAddrRange> &Inits) {
+ return ELFNixPlatformRuntimeState::get().deregisterInits(
+ HeaderAddr, std::move(Inits));
+ })
+ .release();
}
/// Wrapper function for registering metadata on a per-object basis.
diff --git a/compiler-rt/lib/orc/elfnix_platform.h b/compiler-rt/lib/orc/elfnix_platform.h
index 3efac4b2327f32..5ecbdf0cb9c86f 100644
--- a/compiler-rt/lib/orc/elfnix_platform.h
+++ b/compiler-rt/lib/orc/elfnix_platform.h
@@ -37,26 +37,10 @@ struct ELFNixPerObjectSectionsToRegister {
ExecutorAddrRange ThreadDataSection;
};
-struct ELFNixJITDylibInitializers {
- using SectionList = std::vector<ExecutorAddrRange>;
+using ELFNixJITDylibDepInfo = std::vector<ExecutorAddr>;
- ELFNixJITDylibInitializers() = default;
- ELFNixJITDylibInitializers(std::string Name, ExecutorAddr DSOHandleAddress)
- : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {}
-
- std::string Name;
- ExecutorAddr DSOHandleAddress;
-
- std::vector<std::pair<std::string, SectionList>> InitSections;
-};
-
-class ELFNixJITDylibDeinitializers {};
-
-using ELFNixJITDylibInitializerSequence =
- std::vector<ELFNixJITDylibInitializers>;
-
-using ELFNixJITDylibDeinitializerSequence =
- std::vector<ELFNixJITDylibDeinitializers>;
+using ELFNixJITDylibDepInfoMap =
+ std::unordered_map<ExecutorAddr, ELFNixJITDylibDepInfo>;
enum dlopen_mode : int {
ORC_RT_RTLD_LAZY = 0x1,
@@ -94,37 +78,9 @@ class SPSSerializationTraits<SPSELFNixPerObjectSectionsToRegister,
}
};
-using SPSNamedExecutorAddrRangeSequenceMap =
- SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>;
-
-using SPSELFNixJITDylibInitializers =
- SPSTuple<SPSString, SPSExecutorAddr, SPSNamedExecutorAddrRangeSequenceMap>;
-
-using SPSELFNixJITDylibInitializerSequence =
- SPSSequence<SPSELFNixJITDylibInitializers>;
-
-/// Serialization traits for ELFNixJITDylibInitializers.
-template <>
-class SPSSerializationTraits<SPSELFNixJITDylibInitializers,
- elfnix::ELFNixJITDylibInitializers> {
-public:
- static size_t size(const elfnix::ELFNixJITDylibInitializers &MOJDIs) {
- return SPSELFNixJITDylibInitializers::AsArgList::size(
- MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
- }
-
- static bool serialize(SPSOutputBuffer &OB,
- const elfnix::ELFNixJITDylibInitializers &MOJDIs) {
- return SPSELFNixJITDylibInitializers::AsArgList::serialize(
- OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
- }
-
- static bool deserialize(SPSInputBuffer &IB,
- elfnix::ELFNixJITDylibInitializers &MOJDIs) {
- return SPSELFNixJITDylibInitializers::AsArgList::deserialize(
- IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
- }
-};
+using SPSELFNixJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
+using SPSELFNixJITDylibDepInfoMap =
+ SPSSequence<SPSTuple<SPSExecutorAddr, SPSELFNixJITDylibDepInfo>>;
} // namespace orc_rt
diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp
index 8cc3594b5d0cf7..a65e5b94eb4f40 100644
--- a/compiler-rt/lib/orc/macho_platform.cpp
+++ b/compiler-rt/lib/orc/macho_platform.cpp
@@ -17,6 +17,7 @@
#include "error.h"
#include "interval_map.h"
#include "jit_dispatch.h"
+#include "sections_tracker.h"
#include "wrapper_function_utils.h"
#include <algorithm>
@@ -168,93 +169,6 @@ class MachOPlatformRuntimeState {
using AtExitsVector = std::vector<AtExitEntry>;
- /// Used to manage sections of fixed-sized metadata records (e.g. pointer
- /// sections, selector refs, etc.)
- template <typename RecordElement> class RecordSectionsTracker {
- public:
- /// Add a section to the "new" list.
- void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
-
- /// Returns true if there are new sections to process.
- bool hasNewSections() const { return !New.empty(); }
-
- /// Returns the number of new sections to process.
- size_t numNewSections() const { return New.size(); }
-
- /// Process all new sections.
- template <typename ProcessSectionFunc>
- std::enable_if_t<std::is_void_v<
- std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
- processNewSections(ProcessSectionFunc &&ProcessSection) {
- for (auto &Sec : New)
- ProcessSection(Sec);
- moveNewToProcessed();
- }
-
- /// Proces all new sections with a fallible handler.
- ///
- /// Successfully handled sections will be moved to the Processed
- /// list.
- template <typename ProcessSectionFunc>
- std::enable_if_t<
- std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc,
- span<RecordElement>>>,
- Error>
- processNewSections(ProcessSectionFunc &&ProcessSection) {
- for (size_t I = 0; I != New.size(); ++I) {
- if (auto Err = ProcessSection(New[I])) {
- for (size_t J = 0; J != I; ++J)
- Processed.push_back(New[J]);
- New.erase(New.begin(), New.begin() + I);
- return Err;
- }
- }
- moveNewToProcessed();
- return Error::success();
- }
-
- /// Move all sections back to New for reprocessing.
- void reset() {
- moveNewToProcessed();
- New = std::move(Processed);
- }
-
- /// Remove the section with the given range.
- bool removeIfPresent(ExecutorAddrRange R) {
- if (removeIfPresent(New, R))
- return true;
- return removeIfPresent(Processed, R);
- }
-
- private:
- void moveNewToProcessed() {
- if (Processed.empty())
- Processed = std::move(New);
- else {
- Processed.reserve(Processed.size() + New.size());
- std::copy(New.begin(), New.end(), std::back_inserter(Processed));
- New.clear();
- }
- }
-
- bool removeIfPresent(std::vector<span<RecordElement>> &V,
- ExecutorAddrRange R) {
- auto RI = std::find_if(
- V.rbegin(), V.rend(),
- [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
- return E.data() == RS.data();
- });
- if (RI != V.rend()) {
- V.erase(std::next(RI).base());
- return true;
- }
- return false;
- }
-
- std::vector<span<RecordElement>> Processed;
- std::vector<span<RecordElement>> New;
- };
-
struct UnwindSections {
UnwindSections(const UnwindSectionInfo &USI)
: DwarfSection(USI.DwarfSection.toSpan<char>()),
diff --git a/compiler-rt/lib/orc/sections_tracker.h b/compiler-rt/lib/orc/sections_tracker.h
new file mode 100644
index 00000000000000..10851c47aa49e5
--- /dev/null
+++ b/compiler-rt/lib/orc/sections_tracker.h
@@ -0,0 +1,114 @@
+//===- sections_tracker.h - RecordSectionsTracker utilities for the ORC runtime
+//----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// RecordSectionsTracker: Responsible for managing sections of metadata records
+// with fixed sizes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_SECTIONSTRACKER_H
+#define ORC_RT_SECTIONSTRACKER_H
+
+#include "error.h"
+#include "executor_address.h"
+#include <algorithm>
+#include <vector>
+
+namespace orc_rt {
+
+/// Used to manage sections of fixed-sized metadata records (e.g. pointer
+/// sections, selector refs, etc.)
+template <typename RecordElement> class RecordSectionsTracker {
+public:
+ /// Add a section to the "new" list.
+ void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
+
+ /// Returns true if there are new sections to process.
+ bool hasNewSections() const { return !New.empty(); }
+
+ /// Returns the number of new sections to process.
+ size_t numNewSections() const { return New.size(); }
+
+ /// Process all new sections.
+ template <typename ProcessSectionFunc>
+ std::enable_if_t<std::is_void_v<
+ std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
+ processNewSections(ProcessSectionFunc &&ProcessSection) {
+ for (auto &Sec : New)
+ ProcessSection(Sec);
+ moveNewToProcessed();
+ }
+
+ /// Proces all new sections with a fallible handler.
+ ///
+ /// Successfully handled sections will be moved to the Processed
+ /// list.
+ template <typename ProcessSectionFunc>
+ std::enable_if_t<
+ std::is_same_v<
+ Error, std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>,
+ Error>
+ processNewSections(ProcessSectionFunc &&ProcessSection) {
+ for (size_t I = 0; I != New.size(); ++I) {
+ if (auto Err = ProcessSection(New[I])) {
+ for (size_t J = 0; J != I; ++J)
+ Processed.push_back(New[J]);
+ New.erase(New.begin(), New.begin() + I);
+ return Err;
+ }
+ }
+ moveNewToProcessed();
+ return Error::success();
+ }
+
+ /// Move all sections back to New for reprocessing.
+ void reset() {
+ moveNewToProcessed();
+ New = std::move(Processed);
+ }
+
+ /// Remove the section with the given range.
+ bool removeIfPresent(ExecutorAddrRange R) {
+ if (removeIfPresent(New, R))
+ return true;
+ return removeIfPresent(Processed, R);
+ }
+
+private:
+ void moveNewToProcessed() {
+ if (Processed.empty())
+ Processed = std::move(New);
+ else {
+ Processed.reserve(Processed.size() + New.size());
+ std::copy(New.begin(), New.end(), std::back_inserter(Processed));
+ New.clear();
+ }
+ }
+
+ bool removeIfPresent(std::vector<span<RecordElement>> &V,
+ ExecutorAddrRange R) {
+ auto RI = std::find_if(
+ V.rbegin(), V.rend(),
+ [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
+ return E.data() == RS.data();
+ });
+ if (RI != V.rend()) {
+ V.erase(std::next(RI).base());
+ return true;
+ }
+ return false;
+ }
+
+ std::vector<span<RecordElement>> Processed;
+ std::vector<span<RecordElement>> New;
+};
+
+} // namespace orc_rt
+
+#endif // ORC_RT_SECTIONSTRACKER_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
index 2c5b1104926960..40b85e32720108 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
@@ -21,6 +21,7 @@
#include <future>
#include <thread>
+#include <unordered_map>
#include <vector>
namespace llvm {
@@ -31,25 +32,37 @@ struct ELFPerObjectSectionsToRegister {
ExecutorAddrRange ThreadDataSection;
};
-struct ELFNixJITDylibInitializers {
- using SectionList = std::vector<ExecutorAddrRange>;
+using ELFNixJITDylibDepInfo = std::vector<ExecutorAddr>;
+using ELFNixJITDylibDepInfoMap =
+ std::vector<std::pair<ExecutorAddr, ELFNixJITDylibDepInfo>>;
- ELFNixJITDylibInitializers(std::string Name, ExecutorAddr DSOHandleAddress)
- : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {}
-
- std::string Name;
- ExecutorAddr DSOHandleAddress;
-
- StringMap<SectionList> InitSections;
+struct RuntimeFunction {
+ RuntimeFunction(SymbolStringPtr Name) : Name(std::move(Name)) {}
+ SymbolStringPtr Name;
+ ExecutorAddr Addr;
};
-class ELFNixJITDylibDeinitializers {};
+struct FunctionPairKeyHash {
+ std::size_t
+ operator()(const std::pair<RuntimeFunction *, RuntimeFunction *> &key) const {
+ return std::hash<void *>()(key.first->Addr.toPtr<void *>()) ^
+ std::hash<void *>()(key.second->Addr.toPtr<void *>());
+ }
+};
-using ELFNixJITDylibInitializerSequence =
- std::vector<ELFNixJITDylibInitializers>;
+struct FunctionPairKeyEqual {
+ std::size_t
+ operator()(const std::pair<RuntimeFunction *, RuntimeFunction *> &lhs,
+ const std::pair<RuntimeFunction *, RuntimeFunction *> &rhs) const {
+ return lhs.first == rhs.first && lhs.second == rhs.second;
+ }
+};
-using ELFNixJITDylibDeinitializerSequence =
- std::vector<ELFNixJITDylibDeinitializers>;
+using DeferredRuntimeFnMap = std::unordered_map<
+ std::pair<RuntimeFunction *, RuntimeFunction *>,
+ SmallVector<std::pair<shared::WrapperFunctionCall::ArgDataBufferType,
+ shared::WrapperFunctionCall::ArgDataBufferType>>,
+ FunctionPairKeyHash, FunctionPairKeyEqual>;
/// Mediates between ELFNix initialization and ExecutionSession state.
class ELFNixPlatform : public Platform {
@@ -126,6 +139,23 @@ class ELFNixPlatform : public Platform {
standardRuntimeUtilityAliases();
private:
+ // Data needed for bootstrap only.
+ struct BootstrapInfo {
+ std::mutex Mutex;
+ std::condition_variable CV;
+ size_t ActiveGraphs = 0;
+ ExecutorAddr ELFNixHeaderAddr;
+ DeferredRuntimeFnMap DeferredRTFnMap;
+
+ void addArgumentsToRTFnMap(
+ RuntimeFunction *func1, RuntimeFunction *func2,
+ const shared::WrapperFunctionCall::ArgDataBufferType &arg1,
+ const shared::WrapperFunctionCall::ArgDataBufferType &arg2) {
+ auto &argList = DeferredRTFnMap[std::make_pair(func1, func2)];
+ argList.emplace_back(arg1, arg2);
+ }
+ };
+
// The ELFNixPlatformPlugin scans/modifies LinkGraphs to support ELF
// platform features including initializers, exceptions, TLV, and language
// runtime registration.
@@ -151,19 +181,22 @@ class ELFNixPlatform : public Platform {
ResourceKey SrcKey) override {}
private:
- void addInitializerSupportPasses(MaterializationResponsibility &MR,
- jitlink::PassConfiguration &Config);
+ Error bootstrapPipelineStart(jitlink::LinkGraph &G);
+ Error bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G);
+ Error bootstrapPipelineEnd(jitlink::LinkGraph &G);
void addDSOHandleSupportPasses(MaterializationResponsibility &MR,
jitlink::PassConfiguration &Config);
void addEHAndTLVSupportPasses(MaterializationResponsibility &MR,
- jitlink::PassConfiguration &Config);
+ jitlink::PassConfiguration &Config,
+ bool IsBootstrapping);
Error preserveInitSections(jitlink::LinkGraph &G,
MaterializationResponsibility &MR);
- Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD);
+ Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD,
+ bool IsBootstrapping);
Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD);
@@ -171,11 +204,8 @@ class ELFNixPlatform : public Platform {
ELFNixPlatform &MP;
};
- using SendInitializerSequenceFn =
- unique_function<void(Expected<ELFNixJITDylibInitializerSequence>)>;
-
- using SendDeinitializerSequenceFn =
- unique_function<void(Expected<ELFNixJITDylibDeinitializerSequence>)>;
+ using PushInitializersSendResultFn =
+ unique_function<void(Expected<ELFNixJITDylibDepInfoMap>)>;
using SendSymbolAddressFn = unique_function<void(Expected<ExecutorAddr>)>;
@@ -189,53 +219,58 @@ class ELFNixPlatform : public Platform {
// Associate ELFNixPlatform JIT-side runtime support functions with handlers.
Error associateRuntimeSupportFunctions(JITDylib &PlatformJD);
- void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult,
- JITDylib &JD,
- std::vector<JITDylibSP> DFSLinkOrder);
+ void pushInitializersLoop(PushInitializersSendResultFn SendResult,
+ JITDylibSP JD);
- void getInitializersLookupPhase(SendInitializerSequenceFn SendResult,
- JITDylib &JD);
-
- void rt_getInitializers(SendInitializerSequenceFn SendResult,
- StringRef JDName);
-
- void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult,
- ExecutorAddr Handle);
+ void rt_recordInitializers(PushInitializersSendResultFn SendResult,
+ ExecutorAddr JDHeader);
void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle,
StringRef SymbolName);
- // Records the addresses of runtime symbols used by the platform.
- Error bootstrapELFNixRuntime(JITDylib &PlatformJD);
-
- Error registerInitInfo(JITDylib &JD,
- ArrayRef<jitlink::Section *> InitSections);
-
- Error registerPerObjectSections(const ELFPerObjectSectionsToRegister &POSR);
+ Error registerPerObjectSections(jitlink::LinkGraph &G,
+ const ELFPerObjectSectionsToRegister &POSR,
+ bool IsBootstrapping);
Expected<uint64_t> createPThreadKey();
ExecutionSession &ES;
+ JITDylib &PlatformJD;
ObjectLinkingLayer &ObjLinkingLayer;
SymbolStringPtr DSOHandleSymbol;
- std::atomic<bool> RuntimeBootstrapped{false};
- ExecutorAddr orc_rt_elfnix_platform_bootstrap;
- ExecutorAddr orc_rt_elfnix_platform_shutdown;
- ExecutorAddr orc_rt_elfnix_register_object_sections;
- ExecutorAddr orc_rt_elfnix_create_pthread_key;
+ RuntimeFunction PlatformBootstrap{
+ ES.intern("__orc_rt_elfnix_platform_bootstrap")};
+ RuntimeFunction PlatformShutdown{
+ ES.intern("__orc_rt_elfnix_platform_shutdown")};
+ RuntimeFunction RegisterJITDylib{
+ ES.intern("__orc_rt_elfnix_register_jitdylib")};
+ RuntimeFunction DeregisterJITDylib{
+ ES.intern("__orc_rt_elfnix_deregister_jitdylib")};
+ RuntimeFunction RegisterObjectSections{
+ ES.intern("__orc_rt_elfnix_register_object_sections")};
+ RuntimeFunction DeregisterObjectSections{
+ ES.intern("__orc_rt_elfnix_deregister_object_sections")};
+ RuntimeFunction RegisterInitSections{
+ ES.intern("__orc_rt_elfnix_register_init_sections")};
+ RuntimeFunction DeregisterInitSections{
+ ES.intern("__orc_rt_elfnix_deregister_init_sections")};
+ RuntimeFunction CreatePThreadKey{
+ ES.intern("__orc_rt_elfnix_create_pthread_key")};
DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols;
// InitSeqs gets its own mutex to avoid locking the whole session when
// aggregating data from the jitlink.
std::mutex PlatformMutex;
- DenseMap<JITDylib *, ELFNixJITDylibInitializers> InitSeqs;
std::vector<ELFPerObjectSectionsToRegister> BootstrapPOSRs;
DenseMap<ExecutorAddr, JITDylib *> HandleAddrToJITDylib;
+ DenseMap<JITDylib *, ExecutorAddr> JITDylibToHandleAddr;
DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey;
+
+ std::atomic<BootstrapInfo *> Bootstrap;
};
namespace shared {
@@ -266,63 +301,11 @@ class SPSSerializationTraits<SPSELFPerObjectSectionsToRegister,
}
};
-using SPSNamedExecutorAddrRangeSequenceMap =
- SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>;
-
-using SPSELFNixJITDylibInitializers =
- SPSTuple<SPSString, SPSExecutorAddr, SPSNamedExecutorAddrRangeSequenceMap>;
-
-using SPSELFNixJITDylibInitializerSequence =
- SPSSequence<SPSELFNixJITDylibInitializers>;
-
-/// Serialization traits for ELFNixJITDylibInitializers.
-template <>
-class SPSSerializationTraits<SPSELFNixJITDylibInitializers,
- ELFNixJITDylibInitializers> {
-public:
- static size_t size(const ELFNixJITDylibInitializers &MOJDIs) {
- return SPSELFNixJITDylibInitializers::AsArgList::size(
- MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
- }
-
- static bool serialize(SPSOutputBuffer &OB,
- const ELFNixJITDylibInitializers &MOJDIs) {
- return SPSELFNixJITDylibInitializers::AsArgList::serialize(
- OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
- }
-
- static bool deserialize(SPSInputBuffer &IB,
- ELFNixJITDylibInitializers &MOJDIs) {
- return SPSELFNixJITDylibInitializers::AsArgList::deserialize(
- IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
- }
-};
-
-using SPSELFJITDylibDeinitializers = SPSEmpty;
-
-using SPSELFJITDylibDeinitializerSequence =
- SPSSequence<SPSELFJITDylibDeinitializers>;
-
-template <>
-class SPSSerializationTraits<SPSELFJITDylibDeinitializers,
- ELFNixJITDylibDeinitializers> {
-public:
- static size_t size(const ELFNixJITDylibDeinitializers &MOJDDs) { return 0; }
-
- static bool serialize(SPSOutputBuffer &OB,
- const ELFNixJITDylibDeinitializers &MOJDDs) {
- return true;
- }
-
- static bool deserialize(SPSInputBuffer &IB,
- ELFNixJITDylibDeinitializers &MOJDDs) {
- MOJDDs = ELFNixJITDylibDeinitializers();
- return true;
- }
-};
+using SPSELFNixJITDylibDepInfoMap =
+ SPSSequence<SPSTuple<SPSExecutorAddr, SPSSequence<SPSExecutorAddr>>>;
} // end namespace shared
} // end namespace orc
} // end namespace llvm
-#endif // LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H
+#endif // LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H
\ No newline at end of file
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index 67c920a40ea2e5..d92077dbcbd034 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -1,4 +1,5 @@
-//===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
+//===------ ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc
+//-----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -15,6 +16,7 @@
#include "llvm/ExecutionEngine/JITLink/x86_64.h"
#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
#include "llvm/Support/BinaryByteStream.h"
#include "llvm/Support/Debug.h"
@@ -28,6 +30,125 @@ using namespace llvm::orc::shared;
namespace {
+template <typename SPSSerializer, typename... ArgTs>
+shared::WrapperFunctionCall::ArgDataBufferType
+getArgDataBufferType(const ArgTs &...Args) {
+ shared::WrapperFunctionCall::ArgDataBufferType ArgData;
+ ArgData.resize(SPSSerializer::size(Args...));
+ SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
+ ArgData.size());
+ if (SPSSerializer::serialize(OB, Args...))
+ return ArgData;
+ return {};
+}
+
+std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
+ std::string Name) {
+ unsigned PointerSize;
+ llvm::endianness Endianness;
+ const auto &TT = MOP.getExecutionSession().getTargetTriple();
+
+ switch (TT.getArch()) {
+ case Triple::x86_64:
+ PointerSize = 8;
+ Endianness = llvm::endianness::little;
+ break;
+ case Triple::aarch64:
+ PointerSize = 8;
+ Endianness = llvm::endianness::little;
+ break;
+ case Triple::ppc64:
+ PointerSize = 8;
+ Endianness = llvm::endianness::big;
+ break;
+ case Triple::ppc64le:
+ PointerSize = 8;
+ Endianness = llvm::endianness::little;
+ break;
+ default:
+ llvm_unreachable("Unrecognized architecture");
+ }
+
+ return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
+ Endianness,
+ jitlink::getGenericEdgeKindName);
+}
+
+// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
+class ELFNixPlatformCompleteBootstrapMaterializationUnit
+ : public MaterializationUnit {
+public:
+ ELFNixPlatformCompleteBootstrapMaterializationUnit(
+ ELFNixPlatform &MOP, StringRef PlatformJDName,
+ SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
+ ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
+ ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
+ ExecutorAddr DeregisterJITDylib)
+ : MaterializationUnit(
+ {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
+ MOP(MOP), PlatformJDName(PlatformJDName),
+ CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
+ DeferredAAsMap(std::move(DeferredAAs)),
+ ELFNixHeaderAddr(ELFNixHeaderAddr),
+ PlatformBootstrap(PlatformBootstrap),
+ PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
+ DeregisterJITDylib(DeregisterJITDylib) {}
+
+ StringRef getName() const override {
+ return "ELFNixPlatformCompleteBootstrap";
+ }
+
+ void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+ using namespace jitlink;
+ auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
+ auto &PlaceholderSection =
+ G->createSection("__orc_rt_cplt_bs", MemProt::Read);
+ auto &PlaceholderBlock =
+ G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
+ G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
+ Linkage::Strong, Scope::Hidden, false, true);
+
+ // 1. Bootstrap the platform support code.
+ G->allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
+ PlatformBootstrap, ELFNixHeaderAddr)),
+ cantFail(
+ WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
+
+ // 2. Register the platform JITDylib.
+ G->allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<
+ SPSArgList<SPSString, SPSExecutorAddr>>(
+ RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),
+ cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
+ DeregisterJITDylib, ELFNixHeaderAddr))});
+
+ // 4. Add the deferred actions to the graph.
+ for (auto &[Fn, CallDatas] : DeferredAAsMap) {
+ for (auto &CallData : CallDatas) {
+ G->allocActions().push_back(
+ {WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
+ WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
+ }
+ }
+
+ MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
+ }
+
+ void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
+
+private:
+ ELFNixPlatform &MOP;
+ StringRef PlatformJDName;
+ SymbolStringPtr CompleteBootstrapSymbol;
+ DeferredRuntimeFnMap DeferredAAsMap;
+ ExecutorAddr ELFNixHeaderAddr;
+ ExecutorAddr PlatformBootstrap;
+ ExecutorAddr PlatformShutdown;
+ ExecutorAddr RegisterJITDylib;
+ ExecutorAddr DeregisterJITDylib;
+};
+
class DSOHandleMaterializationUnit : public MaterializationUnit {
public:
DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
@@ -174,16 +295,28 @@ ELFNixPlatform::Create(ExecutionSession &ES,
}
Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
- return JD.define(
- std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
+ if (auto Err = JD.define(std::make_unique<DSOHandleMaterializationUnit>(
+ *this, DSOHandleSymbol)))
+ return Err;
+
+ return ES.lookup({&JD}, DSOHandleSymbol).takeError();
}
Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
+ std::lock_guard<std::mutex> Lock(PlatformMutex);
+ auto I = JITDylibToHandleAddr.find(&JD);
+ if (I != JITDylibToHandleAddr.end()) {
+ assert(HandleAddrToJITDylib.count(I->second) &&
+ "HandleAddrToJITDylib missing entry");
+ HandleAddrToJITDylib.erase(I->second);
+ JITDylibToHandleAddr.erase(I);
+ }
return Error::success();
}
Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
const MaterializationUnit &MU) {
+
auto &JD = RT.getJITDylib();
const auto &InitSym = MU.getInitializerSymbol();
if (!InitSym)
@@ -262,14 +395,16 @@ ELFNixPlatform::ELFNixPlatform(
ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
JITDylib &PlatformJD,
std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
- : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
+ : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer),
DSOHandleSymbol(ES.intern("__dso_handle")) {
ErrorAsOutParameter _(&Err);
-
ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
+ BootstrapInfo BI;
+ Bootstrap = &BI;
+
// PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
// the platform now), so set it up.
if (auto E2 = setupJITDylib(PlatformJD)) {
@@ -277,19 +412,44 @@ ELFNixPlatform::ELFNixPlatform(
return;
}
- RegisteredInitSymbols[&PlatformJD].add(
- DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
-
- // Associate wrapper function tags with JIT-side function implementations.
- if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
- Err = std::move(E2);
+ // Step (2) Request runtime registration functions to trigger
+ // materialization..
+ if ((Err = ES.lookup(
+ makeJITDylibSearchOrder(&PlatformJD),
+ SymbolLookupSet(
+ {PlatformBootstrap.Name, PlatformShutdown.Name,
+ RegisterJITDylib.Name, DeregisterJITDylib.Name,
+ RegisterInitSections.Name, DeregisterInitSections.Name,
+ RegisterObjectSections.Name,
+ DeregisterObjectSections.Name, CreatePThreadKey.Name}))
+ .takeError()))
return;
+
+ // Step (3) Wait for any incidental linker work to complete.
+ {
+ std::unique_lock<std::mutex> Lock(BI.Mutex);
+ BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
+ Bootstrap = nullptr;
}
- // Lookup addresses of runtime functions callable by the platform,
- // call the platform bootstrap function to initialize the platform-state
- // object in the executor.
- if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
+ // Step (4) Add complete-bootstrap materialization unit and request.
+ auto BootstrapCompleteSymbol =
+ ES.intern("__orc_rt_elfnix_complete_bootstrap");
+ if ((Err = PlatformJD.define(
+ std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
+ *this, PlatformJD.getName(), BootstrapCompleteSymbol,
+ std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,
+ PlatformBootstrap.Addr, PlatformShutdown.Addr,
+ RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))
+ return;
+ if ((Err = ES.lookup(makeJITDylibSearchOrder(
+ &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
+ std::move(BootstrapCompleteSymbol))
+ .takeError()))
+ return;
+
+ // Associate wrapper function tags with JIT-side function implementations.
+ if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
Err = std::move(E2);
return;
}
@@ -298,17 +458,11 @@ ELFNixPlatform::ELFNixPlatform(
Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
ExecutionSession::JITDispatchHandlerAssociationMap WFs;
- using GetInitializersSPSSig =
- SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
- WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
- ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
- this, &ELFNixPlatform::rt_getInitializers);
-
- using GetDeinitializersSPSSig =
- SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
- WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
- ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
- this, &ELFNixPlatform::rt_getDeinitializers);
+ using RecordInitializersSPSSig =
+ SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
+ WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =
+ ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
+ this, &ELFNixPlatform::rt_recordInitializers);
using LookupSymbolSPSSig =
SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
@@ -319,110 +473,120 @@ Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
}
-void ELFNixPlatform::getInitializersBuildSequencePhase(
- SendInitializerSequenceFn SendResult, JITDylib &JD,
- std::vector<JITDylibSP> DFSLinkOrder) {
- ELFNixJITDylibInitializerSequence FullInitSeq;
- {
- std::lock_guard<std::mutex> Lock(PlatformMutex);
- for (auto &InitJD : reverse(DFSLinkOrder)) {
- LLVM_DEBUG({
- dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
- << "\" to sequence\n";
- });
- auto ISItr = InitSeqs.find(InitJD.get());
- if (ISItr != InitSeqs.end()) {
- FullInitSeq.emplace_back(std::move(ISItr->second));
- InitSeqs.erase(ISItr);
- }
- }
- }
+void ELFNixPlatform::pushInitializersLoop(
+ PushInitializersSendResultFn SendResult, JITDylibSP JD) {
+ DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
+ DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
+ SmallVector<JITDylib *, 16> Worklist({JD.get()});
- SendResult(std::move(FullInitSeq));
-}
+ ES.runSessionLocked([&]() {
+ while (!Worklist.empty()) {
+ // FIXME: Check for defunct dylibs.
-void ELFNixPlatform::getInitializersLookupPhase(
- SendInitializerSequenceFn SendResult, JITDylib &JD) {
+ auto DepJD = Worklist.back();
+ Worklist.pop_back();
- auto DFSLinkOrder = JD.getDFSLinkOrder();
- if (!DFSLinkOrder) {
- SendResult(DFSLinkOrder.takeError());
- return;
- }
+ // If we've already visited this JITDylib on this iteration then continue.
+ if (JDDepMap.count(DepJD))
+ continue;
- DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
- ES.runSessionLocked([&]() {
- for (auto &InitJD : *DFSLinkOrder) {
- auto RISItr = RegisteredInitSymbols.find(InitJD.get());
+ // Add dep info.
+ auto &DM = JDDepMap[DepJD];
+ DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
+ for (auto &KV : O) {
+ if (KV.first == DepJD)
+ continue;
+ DM.push_back(KV.first);
+ Worklist.push_back(KV.first);
+ }
+ });
+
+ // Add any registered init symbols.
+ auto RISItr = RegisteredInitSymbols.find(DepJD);
if (RISItr != RegisteredInitSymbols.end()) {
- NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
+ NewInitSymbols[DepJD] = std::move(RISItr->second);
RegisteredInitSymbols.erase(RISItr);
}
}
});
- // If there are no further init symbols to look up then move on to the next
- // phase.
+ // If there are no further init symbols to look up then send the link order
+ // (as a list of header addresses) to the caller.
if (NewInitSymbols.empty()) {
- getInitializersBuildSequencePhase(std::move(SendResult), JD,
- std::move(*DFSLinkOrder));
+
+ // To make the list intelligible to the runtime we need to convert all
+ // JITDylib pointers to their header addresses. Only include JITDylibs
+ // that appear in the JITDylibToHandleAddr map (i.e. those that have been
+ // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
+ DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
+ HeaderAddrs.reserve(JDDepMap.size());
+ {
+ std::lock_guard<std::mutex> Lock(PlatformMutex);
+ for (auto &KV : JDDepMap) {
+ auto I = JITDylibToHandleAddr.find(KV.first);
+ if (I != JITDylibToHandleAddr.end())
+ HeaderAddrs[KV.first] = I->second;
+ }
+ }
+
+ // Build the dep info map to return.
+ ELFNixJITDylibDepInfoMap DIM;
+ DIM.reserve(JDDepMap.size());
+ for (auto &KV : JDDepMap) {
+ auto HI = HeaderAddrs.find(KV.first);
+ // Skip unmanaged JITDylibs.
+ if (HI == HeaderAddrs.end())
+ continue;
+ auto H = HI->second;
+ ELFNixJITDylibDepInfo DepInfo;
+ for (auto &Dep : KV.second) {
+ auto HJ = HeaderAddrs.find(Dep);
+ if (HJ != HeaderAddrs.end())
+ DepInfo.push_back(HJ->second);
+ }
+ DIM.push_back(std::make_pair(H, std::move(DepInfo)));
+ }
+ SendResult(DIM);
return;
}
// Otherwise issue a lookup and re-run this phase when it completes.
lookupInitSymbolsAsync(
- [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
+ [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
if (Err)
SendResult(std::move(Err));
else
- getInitializersLookupPhase(std::move(SendResult), JD);
+ pushInitializersLoop(std::move(SendResult), JD);
},
ES, std::move(NewInitSymbols));
}
-void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
- StringRef JDName) {
- LLVM_DEBUG({
- dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
- });
-
- JITDylib *JD = ES.getJITDylibByName(JDName);
- if (!JD) {
- LLVM_DEBUG({
- dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n";
- });
- SendResult(make_error<StringError>("No JITDylib named " + JDName,
- inconvertibleErrorCode()));
- return;
- }
-
- getInitializersLookupPhase(std::move(SendResult), *JD);
-}
-
-void ELFNixPlatform::rt_getDeinitializers(
- SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
- LLVM_DEBUG({
- dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n";
- });
-
- JITDylib *JD = nullptr;
-
+void ELFNixPlatform::rt_recordInitializers(
+ PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
+ JITDylibSP JD;
{
std::lock_guard<std::mutex> Lock(PlatformMutex);
- auto I = HandleAddrToJITDylib.find(Handle);
+ auto I = HandleAddrToJITDylib.find(JDHeaderAddr);
if (I != HandleAddrToJITDylib.end())
JD = I->second;
}
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
+ if (JD)
+ dbgs() << "pushing initializers for " << JD->getName() << "\n";
+ else
+ dbgs() << "No JITDylib for header address.\n";
+ });
+
if (!JD) {
- LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
- SendResult(make_error<StringError>("No JITDylib associated with handle " +
- formatv("{0:x}", Handle),
+ SendResult(make_error<StringError>("No JITDylib with header addr " +
+ formatv("{0:x}", JDHeaderAddr),
inconvertibleErrorCode()));
return;
}
- SendResult(ELFNixJITDylibDeinitializerSequence());
+ pushInitializersLoop(std::move(SendResult), JD);
}
void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
@@ -473,116 +637,98 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
}
-Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
-
- std::pair<const char *, ExecutorAddr *> Symbols[] = {
- {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
- {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
- {"__orc_rt_elfnix_register_object_sections",
- &orc_rt_elfnix_register_object_sections},
- {"__orc_rt_elfnix_create_pthread_key",
- &orc_rt_elfnix_create_pthread_key}};
-
- SymbolLookupSet RuntimeSymbols;
- std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
- for (const auto &KV : Symbols) {
- auto Name = ES.intern(KV.first);
- RuntimeSymbols.add(Name);
- AddrsToRecord.push_back({std::move(Name), KV.second});
- }
-
- auto RuntimeSymbolAddrs = ES.lookup(
- {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
- if (!RuntimeSymbolAddrs)
- return RuntimeSymbolAddrs.takeError();
-
- for (const auto &KV : AddrsToRecord) {
- auto &Name = KV.first;
- assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
- *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress();
- }
-
- auto PJDDSOHandle = ES.lookup(
- {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
- if (!PJDDSOHandle)
- return PJDDSOHandle.takeError();
-
- if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
- orc_rt_elfnix_platform_bootstrap,
- PJDDSOHandle->getAddress().getValue()))
- return Err;
-
- // FIXME: Ordering is fuzzy here. We're probably best off saying
- // "behavior is undefined if code that uses the runtime is added before
- // the platform constructor returns", then move all this to the constructor.
- RuntimeBootstrapped = true;
- std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
- {
- std::lock_guard<std::mutex> Lock(PlatformMutex);
- DeferredPOSRs = std::move(BootstrapPOSRs);
- }
-
- for (auto &D : DeferredPOSRs)
- if (auto Err = registerPerObjectSections(D))
- return Err;
-
+Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
+ jitlink::LinkGraph &G) {
+ // Increment the active graphs count in BootstrapInfo.
+ std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
+ ++MP.Bootstrap.load()->ActiveGraphs;
return Error::success();
}
-Error ELFNixPlatform::registerInitInfo(
- JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
-
- std::unique_lock<std::mutex> Lock(PlatformMutex);
-
- ELFNixJITDylibInitializers *InitSeq = nullptr;
- {
- auto I = InitSeqs.find(&JD);
- if (I == InitSeqs.end()) {
- // If there's no init sequence entry yet then we need to look up the
- // header symbol to force creation of one.
- Lock.unlock();
-
- auto SearchOrder =
- JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
- if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
- return Err;
-
- Lock.lock();
- I = InitSeqs.find(&JD);
- assert(I != InitSeqs.end() &&
- "Entry missing after header symbol lookup?");
+Error ELFNixPlatform::ELFNixPlatformPlugin::
+ bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
+ // Record bootstrap function names.
+ std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
+ {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
+ {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
+ {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
+ {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
+ {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
+ {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
+ {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
+ {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
+ {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
+ {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
+
+ bool RegisterELFNixHeader = false;
+
+ for (auto *Sym : G.defined_symbols()) {
+ for (auto &RTSym : RuntimeSymbols) {
+ if (Sym->hasName() && Sym->getName() == RTSym.first) {
+ if (*RTSym.second)
+ return make_error<StringError>(
+ "Duplicate " + RTSym.first +
+ " detected during ELFNixPlatform bootstrap",
+ inconvertibleErrorCode());
+
+ if (Sym->getName() == *MP.DSOHandleSymbol)
+ RegisterELFNixHeader = true;
+
+ *RTSym.second = Sym->getAddress();
+ }
}
- InitSeq = &I->second;
}
- for (auto *Sec : InitSections) {
- // FIXME: Avoid copy here.
- jitlink::SectionRange R(*Sec);
- InitSeq->InitSections[Sec->getName()].push_back(R.getRange());
+ if (RegisterELFNixHeader) {
+ // If this graph defines the elfnix header symbol then create the internal
+ // mapping between it and PlatformJD.
+ std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+ MP.JITDylibToHandleAddr[&MP.PlatformJD] =
+ MP.Bootstrap.load()->ELFNixHeaderAddr;
+ MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
+ &MP.PlatformJD;
}
return Error::success();
}
+Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
+ jitlink::LinkGraph &G) {
+ std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
+ assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
+ --MP.Bootstrap.load()->ActiveGraphs;
+ // Notify Bootstrap->CV while holding the mutex because the mutex is
+ // also keeping Bootstrap->CV alive.
+ if (MP.Bootstrap.load()->ActiveGraphs == 0)
+ MP.Bootstrap.load()->CV.notify_all();
+ return Error::success();
+}
+
Error ELFNixPlatform::registerPerObjectSections(
- const ELFPerObjectSectionsToRegister &POSR) {
+ jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
+ bool IsBootstrapping) {
+ using SPSRegisterPerObjSectionsArgs =
+ SPSArgList<SPSELFPerObjectSectionsToRegister>;
+
+ if (LLVM_UNLIKELY(IsBootstrapping)) {
+ Bootstrap.load()->addArgumentsToRTFnMap(
+ &RegisterObjectSections, &DeregisterObjectSections,
+ getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR),
+ getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR));
+ return Error::success();
+ }
- if (!orc_rt_elfnix_register_object_sections)
- return make_error<StringError>("Attempting to register per-object "
- "sections, but runtime support has not "
- "been loaded yet",
- inconvertibleErrorCode());
+ G.allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
+ RegisterObjectSections.Addr, POSR)),
+ cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
+ DeregisterObjectSections.Addr, POSR))});
- Error ErrResult = Error::success();
- if (auto Err = ES.callSPSWrapper<shared::SPSError(
- SPSELFPerObjectSectionsToRegister)>(
- orc_rt_elfnix_register_object_sections, ErrResult, POSR))
- return Err;
- return ErrResult;
+ return Error::success();
}
Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
- if (!orc_rt_elfnix_create_pthread_key)
+ if (!CreatePThreadKey.Addr)
return make_error<StringError>(
"Attempting to create pthread key in target, but runtime support has "
"not been loaded yet",
@@ -590,7 +736,7 @@ Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
Expected<uint64_t> Result(0);
if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
- orc_rt_elfnix_create_pthread_key, Result))
+ CreatePThreadKey.Addr, Result))
return std::move(Err);
return Result;
}
@@ -598,38 +744,53 @@ Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
jitlink::PassConfiguration &Config) {
+ using namespace jitlink;
+
+ bool InBootstrapPhase =
+ &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
+
+ // If we're in the bootstrap phase then increment the active graphs.
+ if (InBootstrapPhase) {
+ Config.PrePrunePasses.push_back(
+ [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
+ Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
+ return bootstrapPipelineRecordRuntimeFunctions(G);
+ });
+ }
// If the initializer symbol is the __dso_handle symbol then just add
// the DSO handle support passes.
- if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
- addDSOHandleSupportPasses(MR, Config);
- // The DSOHandle materialization unit doesn't require any other
- // support, so we can bail out early.
- return;
- }
+ if (auto InitSymbol = MR.getInitializerSymbol()) {
+ if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
+ addDSOHandleSupportPasses(MR, Config);
+ // The DSOHandle materialization unit doesn't require any other
+ // support, so we can bail out early.
+ return;
+ }
- // If the object contains initializers then add passes to record them.
- if (MR.getInitializerSymbol())
- addInitializerSupportPasses(MR, Config);
+ /// Preserve init sections.
+ Config.PrePrunePasses.push_back(
+ [this, &MR](jitlink::LinkGraph &G) -> Error {
+ if (auto Err = preserveInitSections(G, MR))
+ return Err;
+ return Error::success();
+ });
+ }
// Add passes for eh-frame and TLV support.
- addEHAndTLVSupportPasses(MR, Config);
-}
+ addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);
-void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
- MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
-
- /// Preserve init sections.
- Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
- if (auto Err = preserveInitSections(G, MR))
- return Err;
- return Error::success();
+ // If the object contains initializers then add passes to record them.
+ Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib(),
+ InBootstrapPhase](jitlink::LinkGraph &G) {
+ return registerInitSections(G, JD, InBootstrapPhase);
});
- Config.PostFixupPasses.push_back(
- [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
- return registerInitSections(G, JD);
- });
+ // If we're in the bootstrap phase then steal allocation actions and then
+ // decrement the active graphs.
+ if (InBootstrapPhase)
+ Config.PostFixupPasses.push_back(
+ [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
}
void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
@@ -645,16 +806,22 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
auto HandleAddr = (*I)->getAddress();
MP.HandleAddrToJITDylib[HandleAddr] = &JD;
- assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
- MP.InitSeqs.insert(std::make_pair(
- &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
+ MP.JITDylibToHandleAddr[&JD] = HandleAddr;
+
+ G.allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<
+ SPSArgList<SPSString, SPSExecutorAddr>>(
+ MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),
+ cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
+ MP.DeregisterJITDylib.Addr, HandleAddr))});
}
return Error::success();
});
}
void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
- MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
+ MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
+ bool IsBootstrapping) {
// Insert TLV lowering at the start of the PostPrunePasses, since we want
// it to run before GOT/PLT lowering.
@@ -668,7 +835,8 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
// Add a pass to register the final addresses of the eh-frame and TLV sections
// with the runtime.
- Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
+ Config.PostFixupPasses.push_back([this, IsBootstrapping](
+ jitlink::LinkGraph &G) -> Error {
ELFPerObjectSectionsToRegister POSR;
if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
@@ -702,17 +870,7 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
}
if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
-
- // If we're still bootstrapping the runtime then just record this
- // frame for now.
- if (!MP.RuntimeBootstrapped) {
- std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
- MP.BootstrapPOSRs.push_back(POSR);
- return Error::success();
- }
-
- // Otherwise register it immediately.
- if (auto Err = MP.registerPerObjectSections(POSR))
+ if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
return Err;
}
@@ -757,28 +915,55 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
}
Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
- jitlink::LinkGraph &G, JITDylib &JD) {
-
- SmallVector<jitlink::Section *> InitSections;
-
+ jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
+ SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
for (auto &Sec : G.sections()) {
if (isELFInitializerSection(Sec.getName())) {
- InitSections.push_back(&Sec);
+ jitlink::SectionRange R(Sec);
+ ELFNixPlatformSecs.push_back(R.getRange());
}
}
// Dump the scraped inits.
LLVM_DEBUG({
dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
- for (auto *Sec : InitSections) {
- jitlink::SectionRange R(*Sec);
- dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n";
+ for (auto &Sec : G.sections()) {
+ jitlink::SectionRange R(Sec);
+ dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
}
});
- return MP.registerInitInfo(JD, InitSections);
+ ExecutorAddr HeaderAddr;
+ {
+ std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+ auto I = MP.JITDylibToHandleAddr.find(&JD);
+ assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
+ assert(I->second && "Null header registered for JD");
+ HeaderAddr = I->second;
+ }
+
+ using SPSRegisterInitSectionsArgs =
+ SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
+
+ if (LLVM_UNLIKELY(IsBootstrapping)) {
+ MP.Bootstrap.load()->addArgumentsToRTFnMap(
+ &MP.RegisterInitSections, &MP.DeregisterInitSections,
+ getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
+ ELFNixPlatformSecs),
+ getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
+ ELFNixPlatformSecs));
+ return Error::success();
+ }
+
+ G.allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
+ MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),
+ cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
+ MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});
+
+ return Error::success();
}
Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
More information about the llvm-commits
mailing list