[compiler-rt] 217fd59 - [ORC] Introduce deferred allocation-actions scheme for MachOPlatform bootstrap.
Lang Hames via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 13 16:16:03 PST 2023
Author: Lang Hames
Date: 2023-01-13T16:15:57-08:00
New Revision: 217fd59cc5849f1afdbb8a74807a80b23f2da227
URL: https://github.com/llvm/llvm-project/commit/217fd59cc5849f1afdbb8a74807a80b23f2da227
DIFF: https://github.com/llvm/llvm-project/commit/217fd59cc5849f1afdbb8a74807a80b23f2da227.diff
LOG: [ORC] Introduce deferred allocation-actions scheme for MachOPlatform bootstrap.
This patch modifies the MachOPlatform bootstrap process to record allocation
actions for ORC runtime platform support code in a "deferred actions" vector
rather than attaching it to the corresponding LinkGraphs up-front. The deferred
allocation-actions are run after all the platform support code has been loaded
by attaching them to a separate "bootstrap-complete" graph.
This change should allow the mach-o platform support code in the ORC runtime to
use advanced mach-o platform features (e.g. static inits, TLVs), provided that
the support code does not use these features at runtime before the bootstrap
process completes, or after the shutdown process starts. This is a nice
improvement in and of itself but is motivated by specific future plans: we
want to start recording unwind info in the mach-o platform state object*, and
the recording functions will have their own frame info that needs registering.
The deferred allocation-actions scheme allows for this.
* The plan is to add a new unwind-info-lookup path to libunwind to allow it to
call back to the ORC runtime to find unwind sections. This will simplify the
implementation of support for JIT'd compact-unwind info.
Added:
Modified:
compiler-rt/lib/orc/CMakeLists.txt
compiler-rt/lib/orc/macho_platform.cpp
llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
Removed:
compiler-rt/lib/orc/macho_ehframe_registration.cpp
################################################################################
diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index 4bb91782046b2..acc37319974c4 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -15,7 +15,6 @@ set(ALL_ORC_SOURCES
coff_platform.cpp
coff_platform.per_jd.cpp
elfnix_platform.cpp
- macho_ehframe_registration.cpp
macho_platform.cpp
)
@@ -96,7 +95,6 @@ if (APPLE)
set(ORC_SOURCES
${ORC_COMMON_SOURCES}
- macho_ehframe_registration.cpp
macho_platform.cpp
)
diff --git a/compiler-rt/lib/orc/macho_ehframe_registration.cpp b/compiler-rt/lib/orc/macho_ehframe_registration.cpp
deleted file mode 100644
index 6e647064d9324..0000000000000
--- a/compiler-rt/lib/orc/macho_ehframe_registration.cpp
+++ /dev/null
@@ -1,74 +0,0 @@
-//===- ehframe_registration.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains code required to load the rest of the MachO runtime.
-//
-//===----------------------------------------------------------------------===//
-
-#include "adt.h"
-#include "common.h"
-#include "executor_address.h"
-#include "orc_rt/c_api.h"
-#include "wrapper_function_utils.h"
-
-using namespace __orc_rt;
-
-// eh-frame registration functions.
-// We expect these to be available for all processes.
-extern "C" void __register_frame(const void *);
-extern "C" void __deregister_frame(const void *);
-
-namespace {
-
-template <typename HandleFDEFn>
-void walkEHFrameSection(span<const char> EHFrameSection,
- HandleFDEFn HandleFDE) {
- const char *CurCFIRecord = EHFrameSection.data();
- uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
-
- while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
- const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
- if (Size == 0xffffffff)
- Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
- else
- Size += 4;
- uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
-
- if (Offset != 0)
- HandleFDE(CurCFIRecord);
-
- CurCFIRecord += Size;
- Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
- }
-}
-
-} // end anonymous namespace
-
-ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
-__orc_rt_macho_register_ehframe_section(char *ArgData, size_t ArgSize) {
- return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle(
- ArgData, ArgSize,
- [](ExecutorAddrRange FrameSection) -> Error {
- walkEHFrameSection(FrameSection.toSpan<const char>(),
- __register_frame);
- return Error::success();
- })
- .release();
-}
-
-ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
-__orc_rt_macho_deregister_ehframe_section(char *ArgData, size_t ArgSize) {
- return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle(
- ArgData, ArgSize,
- [](ExecutorAddrRange FrameSection) -> Error {
- walkEHFrameSection(FrameSection.toSpan<const char>(),
- __deregister_frame);
- return Error::success();
- })
- .release();
-}
diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp
index dd992acb2be4b..7d611e1477bd7 100644
--- a/compiler-rt/lib/orc/macho_platform.cpp
+++ b/compiler-rt/lib/orc/macho_platform.cpp
@@ -284,6 +284,9 @@ class MachOPlatformRuntimeState {
Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
std::string_view Symbol);
+ static Error registerEHFrames(span<const char> EHFrameSection);
+ static Error deregisterEHFrames(span<const char> EHFrameSection);
+
static Error registerObjCSelectors(JITDylibState &JDS);
static Error registerObjCClasses(JITDylibState &JDS);
static Error registerSwift5Protocols(JITDylibState &JDS);
@@ -438,7 +441,10 @@ Error MachOPlatformRuntimeState::registerObjectPlatformSections(
for (auto &KV : Secs) {
// FIXME: Validate section ranges?
- if (KV.first == "__DATA,__data") {
+ if (KV.first == "__TEXT,__eh_frame") {
+ if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))
+ return Err;
+ } else if (KV.first == "__DATA,__data") {
assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
"Address already registered.");
auto S = KV.second.toSpan<char>();
@@ -505,7 +511,10 @@ Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
for (auto &KV : Secs) {
// FIXME: Validate section ranges?
- if (KV.first == "__DATA,__data") {
+ if (KV.first == "__TEXT,__eh_frame") {
+ if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))
+ return Err;
+ } else if (KV.first == "__DATA,__data") {
JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());
} else if (KV.first == "__DATA,__common") {
JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());
@@ -673,6 +682,45 @@ MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
return Result;
}
+// eh-frame registration functions.
+// We expect these to be available for all processes.
+extern "C" void __register_frame(const void *);
+extern "C" void __deregister_frame(const void *);
+
+template <typename HandleFDEFn>
+void walkEHFrameSection(span<const char> EHFrameSection,
+ HandleFDEFn HandleFDE) {
+ const char *CurCFIRecord = EHFrameSection.data();
+ uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
+
+ while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
+ const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
+ if (Size == 0xffffffff)
+ Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
+ else
+ Size += 4;
+ uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
+
+ if (Offset != 0)
+ HandleFDE(CurCFIRecord);
+
+ CurCFIRecord += Size;
+ Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
+ }
+}
+
+Error MachOPlatformRuntimeState::registerEHFrames(
+ span<const char> EHFrameSection) {
+ walkEHFrameSection(EHFrameSection, __register_frame);
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::deregisterEHFrames(
+ span<const char> EHFrameSection) {
+ walkEHFrameSection(EHFrameSection, __deregister_frame);
+ return Error::success();
+}
+
Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) {
if (!JDS.ObjCSelRefsSections.hasNewSections())
return Error::success();
@@ -1050,14 +1098,24 @@ Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
- MachOPlatformRuntimeState::initialize();
- return WrapperFunctionResult().release();
+ return WrapperFunction<SPSError()>::handle(
+ ArgData, ArgSize,
+ []() -> Error {
+ MachOPlatformRuntimeState::initialize();
+ return Error::success();
+ })
+ .release();
}
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
- MachOPlatformRuntimeState::destroy();
- return WrapperFunctionResult().release();
+ return WrapperFunction<SPSError()>::handle(
+ ArgData, ArgSize,
+ []() -> Error {
+ MachOPlatformRuntimeState::destroy();
+ return Error::success();
+ })
+ .release();
}
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
index 6c67889b2cf7e..051ab3a85935f 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
@@ -107,6 +107,15 @@ class MachOPlatform : public Platform {
static bool isInitializerSection(StringRef SegName, StringRef SectName);
private:
+ // Data needed for bootstrap only.
+ struct BootstrapInfo {
+ std::mutex Mutex;
+ std::condition_variable CV;
+ size_t ActiveGraphs = 0;
+ shared::AllocActions DeferredAAs;
+ ExecutorAddr MachOHeaderAddr;
+ };
+
// The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO
// platform features including initializers, exceptions, TLV, and language
// runtime registration.
@@ -138,6 +147,12 @@ class MachOPlatform : public Platform {
using InitSymbolDepMap =
DenseMap<MaterializationResponsibility *, JITLinkSymbolSet>;
+ Error bootstrapPipelineStart(jitlink::LinkGraph &G);
+ Error bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G);
+ Error bootstrapPipelineEnd(jitlink::LinkGraph &G);
+
+ Error recordRuntimeRegistrationFunctions(jitlink::LinkGraph &G);
+
Error associateJITDylibHeaderSymbol(jitlink::LinkGraph &G,
MaterializationResponsibility &MR);
@@ -149,9 +164,8 @@ class MachOPlatform : public Platform {
Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD);
- Error registerObjectPlatformSections(jitlink::LinkGraph &G, JITDylib &JD);
-
- Error registerEHSectionsPhase1(jitlink::LinkGraph &G);
+ Error registerObjectPlatformSections(jitlink::LinkGraph &G, JITDylib &JD,
+ bool InBootstrapPhase);
std::mutex PluginMutex;
MachOPlatform &MP;
@@ -179,7 +193,7 @@ class MachOPlatform : public Platform {
Error &Err);
// Associate MachOPlatform JIT-side runtime support functions with handlers.
- Error associateRuntimeSupportFunctions(JITDylib &PlatformJD);
+ Error associateRuntimeSupportFunctions();
// Implements rt_pushInitializers by making repeat async lookups for
// initializer symbols (each lookup may spawn more initializer symbols if
@@ -195,19 +209,14 @@ class MachOPlatform : public Platform {
void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle,
StringRef SymbolName);
- // Records the addresses of runtime symbols used by the platform.
- Error bootstrapMachORuntime(JITDylib &PlatformJD);
-
// Call the ORC runtime to create a pthread key.
Expected<uint64_t> createPThreadKey();
- enum PlatformState { BootstrapPhase1, BootstrapPhase2, Initialized };
-
ExecutionSession &ES;
+ JITDylib &PlatformJD;
ObjectLinkingLayer &ObjLinkingLayer;
- SymbolStringPtr MachOHeaderStartSymbol;
- std::atomic<PlatformState> State{BootstrapPhase1};
+ SymbolStringPtr MachOHeaderStartSymbol = ES.intern("___dso_handle");
struct RuntimeFunction {
RuntimeFunction(SymbolStringPtr Name) : Name(std::move(Name)) {}
@@ -240,6 +249,8 @@ class MachOPlatform : public Platform {
DenseMap<JITDylib *, ExecutorAddr> JITDylibToHeaderAddr;
DenseMap<ExecutorAddr, JITDylib *> HeaderAddrToJITDylib;
DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey;
+
+ std::atomic<BootstrapInfo *> Bootstrap;
};
namespace shared {
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index d85e8944af2b8..a926222dc458f 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -58,6 +58,29 @@ class SPSSerializationTraits<SPSMachOJITDylibDepInfo,
namespace {
+std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP,
+ std::string Name) {
+ unsigned PointerSize;
+ support::endianness Endianness;
+ const auto &TT =
+ MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
+
+ switch (TT.getArch()) {
+ case Triple::aarch64:
+ case Triple::x86_64:
+ PointerSize = 8;
+ Endianness = support::endianness::little;
+ break;
+ default:
+ llvm_unreachable("Unrecognized architecture");
+ }
+
+ return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
+ Endianness,
+ jitlink::getGenericEdgeKindName);
+}
+
+// Generates a MachO header.
class MachOHeaderMaterializationUnit : public MaterializationUnit {
public:
MachOHeaderMaterializationUnit(MachOPlatform &MOP,
@@ -68,41 +91,28 @@ class MachOHeaderMaterializationUnit : public MaterializationUnit {
StringRef getName() const override { return "MachOHeaderMU"; }
void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
- unsigned PointerSize;
- support::endianness Endianness;
- const auto &TT =
- MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
+ auto G = createPlatformGraph(MOP, "<MachOHeaderMU>");
+ addMachOHeader(*G, MOP, R->getInitializerSymbol());
+ MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
+ }
- switch (TT.getArch()) {
- case Triple::aarch64:
- case Triple::x86_64:
- PointerSize = 8;
- Endianness = support::endianness::little;
- break;
- default:
- llvm_unreachable("Unrecognized architecture");
- }
+ void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
- auto G = std::make_unique<jitlink::LinkGraph>(
- "<MachOHeaderMU>", TT, PointerSize, Endianness,
- jitlink::getGenericEdgeKindName);
- auto &HeaderSection = G->createSection("__header", MemProt::Read);
- auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
+ static void addMachOHeader(jitlink::LinkGraph &G, MachOPlatform &MOP,
+ const SymbolStringPtr &InitializerSymbol) {
+ auto &HeaderSection = G.createSection("__header", MemProt::Read);
+ auto &HeaderBlock = createHeaderBlock(G, HeaderSection);
// Init symbol is header-start symbol.
- G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(),
- HeaderBlock.getSize(), jitlink::Linkage::Strong,
- jitlink::Scope::Default, false, true);
+ G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol,
+ HeaderBlock.getSize(), jitlink::Linkage::Strong,
+ jitlink::Scope::Default, false, true);
for (auto &HS : AdditionalHeaderSymbols)
- G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name,
- HeaderBlock.getSize(), jitlink::Linkage::Strong,
- jitlink::Scope::Default, false, true);
-
- MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
+ G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(),
+ jitlink::Linkage::Strong, jitlink::Scope::Default,
+ false, true);
}
- void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
-
private:
struct HeaderSymbol {
const char *Name;
@@ -164,6 +174,78 @@ class MachOHeaderMaterializationUnit : public MaterializationUnit {
constexpr MachOHeaderMaterializationUnit::HeaderSymbol
MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[];
+// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
+class MachOPlatformCompleteBootstrapMaterializationUnit
+ : public MaterializationUnit {
+public:
+ MachOPlatformCompleteBootstrapMaterializationUnit(
+ MachOPlatform &MOP, StringRef PlatformJDName,
+ SymbolStringPtr CompleteBootstrapSymbol, shared::AllocActions DeferredAAs,
+ ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown,
+ ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib,
+ ExecutorAddr MachOHeaderAddr)
+ : MaterializationUnit(
+ {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
+ MOP(MOP), PlatformJDName(PlatformJDName),
+ CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
+ DeferredAAs(std::move(DeferredAAs)),
+ PlatformBootstrap(PlatformBootstrap),
+ PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
+ DeregisterJITDylib(DeregisterJITDylib),
+ MachOHeaderAddr(MachOHeaderAddr) {}
+
+ StringRef getName() const override {
+ return "MachOPlatformCompleteBootstrap";
+ }
+
+ void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+ using namespace jitlink;
+ auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
+ auto &PlaceholderSection =
+ G->createSection("__orc_rt_cplt_bs", MemProt::Read);
+ auto &PlaceholderBlock =
+ G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
+ G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
+ Linkage::Strong, Scope::Hidden, false, true);
+
+ // Reserve space for the stolen actions, plus two extras.
+ G->allocActions().reserve(DeferredAAs.size() + 2);
+
+ // 1. Bootstrap the platform support code.
+ G->allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(PlatformBootstrap)),
+ cantFail(
+ WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
+
+ // 2. Register the platform JITDylib.
+ G->allocActions().push_back(
+ {cantFail(WrapperFunctionCall::Create<
+ SPSArgList<SPSString, SPSExecutorAddr>>(
+ RegisterJITDylib, PlatformJDName, MachOHeaderAddr)),
+ cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
+ DeregisterJITDylib, MachOHeaderAddr))});
+
+ // 3. Add the deferred actions to the graph.
+ std::move(DeferredAAs.begin(), DeferredAAs.end(),
+ std::back_inserter(G->allocActions()));
+
+ MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
+ }
+
+ void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
+
+private:
+ MachOPlatform &MOP;
+ StringRef PlatformJDName;
+ SymbolStringPtr CompleteBootstrapSymbol;
+ shared::AllocActions DeferredAAs;
+ ExecutorAddr PlatformBootstrap;
+ ExecutorAddr PlatformShutdown;
+ ExecutorAddr RegisterJITDylib;
+ ExecutorAddr DeregisterJITDylib;
+ ExecutorAddr MachOHeaderAddr;
+};
+
StringRef DataCommonSectionName = "__DATA,__common";
StringRef DataDataSectionName = "__DATA,__data";
StringRef EHFrameSectionName = "__TEXT,__eh_frame";
@@ -338,50 +420,115 @@ MachOPlatform::MachOPlatform(
ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
JITDylib &PlatformJD,
std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
- : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
- MachOHeaderStartSymbol(ES.intern("___dso_handle")) {
+ : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer) {
ErrorAsOutParameter _(&Err);
-
ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
-
PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
- // Force linking of eh-frame registration functions.
- if (auto Err2 = lookupAndRecordAddrs(
- ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
- {{RegisterEHFrameSection.Name, &RegisterEHFrameSection.Addr},
- {DeregisterEHFrameSection.Name, &DeregisterEHFrameSection.Addr}})) {
- Err = std::move(Err2);
+ BootstrapInfo BI;
+ Bootstrap = &BI;
+
+ // Bootstrap process -- here be phase-ordering dragons.
+ //
+ // The MachOPlatform class uses allocation actions to register metadata
+ // sections with the ORC runtime, however the runtime contains metadata
+ // registration functions that have their own metadata that they need to
+ // register (e.g. the frame-info registration functions have frame-info).
+ // We can't use an ordinary lookup to find these registration functions
+ // because their address is needed during the link of the containing graph
+ // itself (to build the allocation actions that will call the registration
+ // functions). Further complicating the situation (a) the graph containing
+ // the registration functions is allowed to depend on other graphs (e.g. the
+ // graph containing the ORC runtime RTTI support) so we need to handle with
+ // an unknown set of dependencies during bootstrap, and (b) these graphs may
+ // be linked concurrently if the user has installed a concurrent dispatcher.
+ //
+ // We satisfy these constraint by implementing a bootstrap phase during which
+ // allocation actions generated by MachOPlatform are appended to a list of
+ // deferred allocation actions, rather than to the graphs themselves. At the
+ // end of the bootstrap process the deferred actions are attached to a final
+ // "complete-bootstrap" graph that causes them to be run.
+ //
+ // The bootstrap steps are as follows:
+ //
+ // 1. Request the graph containing the mach header. This graph is guaranteed
+ // not to have any metadata so the fact that the registration functions
+ // are not available yet is not a problem.
+ //
+ // 2. Look up the registration functions and discard the results. This will
+ // trigger linking of the graph containing these functions, and
+ // consequently any graphs that it depends on. We do not use the lookup
+ // result to find the addresses of the functions requested (as described
+ // above the lookup will return too late for that), instead we capture the
+ // addresses in a post-allocation pass injected by the platform runtime
+ // during bootstrap only.
+ //
+ // 3. During bootstrap the MachOPlatformPlugin keeps a count of the number of
+ // graphs being linked (potentially concurrently), and we block until all
+ // of these graphs have completed linking. This is to avoid a race on the
+ // deferred-actions vector: the lookup for the runtime registration
+ // functions may return while some functions (those that are being
+ // incidentally linked in, but aren't reachable via the runtime functions)
+ // are still being linked, and we need to capture any allocation actions
+ // for this incidental code before we proceed.
+ //
+ // 4. Once all active links are complete we transfer the deferred actions to
+ // a newly added CompleteBootstrap graph and then request a symbol from
+ // the CompleteBootstrap graph to trigger materialization. This will cause
+ // all deferred actions to be run, and once this lookup returns we can
+ // proceed.
+ //
+ // 5. Finally, we associate runtime support methods in MachOPlatform with
+ // the corresponding jit-dispatch tag variables in the ORC runtime to make
+ // the support methods callable. The bootstrap is now complete.
+
+ // Step (1) Add header materialization unit and request.
+ if ((Err = PlatformJD.define(std::make_unique<MachOHeaderMaterializationUnit>(
+ *this, MachOHeaderStartSymbol))))
return;
- }
-
- State = BootstrapPhase2;
-
- // Associate wrapper function tags with JIT-side function implementations.
- if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
- Err = std::move(E2);
+ if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError()))
return;
- }
- // Lookup addresses of runtime functions callable by the platform,
- // call the platform bootstrap function to initialize the platform-state
- // object in the executor.
- if (auto E2 = bootstrapMachORuntime(PlatformJD)) {
- Err = std::move(E2);
+ // Step (2) Request runtime registration functions to trigger
+ // materialization..
+ if ((Err = ES.lookup(makeJITDylibSearchOrder(&PlatformJD),
+ SymbolLookupSet(
+ {PlatformBootstrap.Name, PlatformShutdown.Name,
+ RegisterJITDylib.Name, DeregisterJITDylib.Name,
+ RegisterObjectPlatformSections.Name,
+ DeregisterObjectPlatformSections.Name,
+ CreatePThreadKey.Name}))
+ .takeError()))
return;
+
+ // Step (3) Wait for any incidental linker work to complete.
+ {
+ std::unique_lock<std::mutex> Lock(BI.Mutex);
+ BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
+ Bootstrap = nullptr;
}
- // PlatformJD hasn't been set up by the platform yet (since we're creating
- // the platform now), so set it up.
- if (auto E2 = setupJITDylib(PlatformJD)) {
- Err = std::move(E2);
+ // Step (4) Add complete-bootstrap materialization unit and request.
+ auto BootstrapCompleteSymbol = ES.intern("__orc_rt_macho_complete_bootstrap");
+ if ((Err = PlatformJD.define(
+ std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>(
+ *this, PlatformJD.getName(), BootstrapCompleteSymbol,
+ std::move(BI.DeferredAAs), PlatformBootstrap.Addr,
+ PlatformShutdown.Addr, RegisterJITDylib.Addr,
+ DeregisterJITDylib.Addr, BI.MachOHeaderAddr))))
+ return;
+ if ((Err = ES.lookup(makeJITDylibSearchOrder(
+ &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
+ std::move(BootstrapCompleteSymbol))
+ .takeError()))
return;
- }
- State = Initialized;
+ // (5) Associate runtime support functions.
+ if ((Err = associateRuntimeSupportFunctions()))
+ return;
}
-Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
+Error MachOPlatform::associateRuntimeSupportFunctions() {
ExecutionSession::JITDispatchHandlerAssociationMap WFs;
using PushInitializersSPSSig =
@@ -569,23 +716,6 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
}
-Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) {
- if (auto Err = lookupAndRecordAddrs(
- ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
- {{PlatformBootstrap.Name, &PlatformBootstrap.Addr},
- {PlatformShutdown.Name, &PlatformShutdown.Addr},
- {RegisterJITDylib.Name, &RegisterJITDylib.Addr},
- {DeregisterJITDylib.Name, &DeregisterJITDylib.Addr},
- {RegisterObjectPlatformSections.Name,
- &RegisterObjectPlatformSections.Addr},
- {DeregisterObjectPlatformSections.Name,
- &DeregisterObjectPlatformSections.Addr},
- {CreatePThreadKey.Name, &CreatePThreadKey.Addr}}))
- return Err;
-
- return ES.callSPSWrapper<void()>(PlatformBootstrap.Addr);
-}
-
Expected<uint64_t> MachOPlatform::createPThreadKey() {
if (!CreatePThreadKey.Addr)
return make_error<StringError>(
@@ -604,7 +734,19 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
jitlink::PassConfiguration &Config) {
- auto PS = MP.State.load();
+ using namespace jitlink;
+
+ bool InBootstrapPhase =
+ &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
+
+ // If we're in the bootstrap phase then increment the active graphs.
+ if (InBootstrapPhase) {
+ Config.PrePrunePasses.push_back(
+ [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
+ Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
+ return bootstrapPipelineRecordRuntimeFunctions(G);
+ });
+ }
// --- Handle Initializers ---
if (auto InitSymbol = MR.getInitializerSymbol()) {
@@ -612,8 +754,8 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
// If the initializer symbol is the MachOHeader start symbol then just
// register it and then bail out -- the header materialization unit
// definitely doesn't need any other passes.
- if (InitSymbol == MP.MachOHeaderStartSymbol) {
- Config.PostAllocationPasses.push_back([this, &MR](jitlink::LinkGraph &G) {
+ if (InitSymbol == MP.MachOHeaderStartSymbol && !InBootstrapPhase) {
+ Config.PostAllocationPasses.push_back([this, &MR](LinkGraph &G) {
return associateJITDylibHeaderSymbol(G, MR);
});
return;
@@ -622,34 +764,33 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
// If the object contains an init symbol other than the header start symbol
// then add passes to preserve, process and register the init
// sections/symbols.
- Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
+ Config.PrePrunePasses.push_back([this, &MR](LinkGraph &G) {
if (auto Err = preserveInitSections(G, MR))
return Err;
return processObjCImageInfo(G, MR);
});
}
- // --- Add passes for eh-frame and TLV support ---
- if (PS == MachOPlatform::BootstrapPhase1) {
- Config.PostFixupPasses.push_back(
- [this](jitlink::LinkGraph &G) { return registerEHSectionsPhase1(G); });
- return;
- }
-
// Insert TLV lowering at the start of the PostPrunePasses, since we want
// it to run before GOT/PLT lowering.
Config.PostPrunePasses.insert(
Config.PostPrunePasses.begin(),
- [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
+ [this, &JD = MR.getTargetJITDylib()](LinkGraph &G) {
return fixTLVSectionsAndEdges(G, JD);
});
// Add a pass to register the final addresses of any special sections in the
// object with the runtime.
Config.PostAllocationPasses.push_back(
- [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
- return registerObjectPlatformSections(G, JD);
+ [this, &JD = MR.getTargetJITDylib(), InBootstrapPhase](LinkGraph &G) {
+ return registerObjectPlatformSections(G, JD, InBootstrapPhase);
});
+
+ // If we're in the bootstrap phase then steal allocation actions and then
+ // decrement the active graphs.
+ if (InBootstrapPhase)
+ Config.PostFixupPasses.push_back(
+ [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
}
ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
@@ -666,6 +807,73 @@ MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
return SyntheticSymbolDependenciesMap();
}
+Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineStart(
+ jitlink::LinkGraph &G) {
+ // Increment the active graphs count in BootstrapInfo.
+ std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
+ ++MP.Bootstrap.load()->ActiveGraphs;
+ return Error::success();
+}
+
+Error MachOPlatform::MachOPlatformPlugin::
+ bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
+ // Record bootstrap function names.
+ std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
+ {*MP.MachOHeaderStartSymbol, &MP.Bootstrap.load()->MachOHeaderAddr},
+ {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
+ {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
+ {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
+ {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
+ {*MP.RegisterObjectPlatformSections.Name,
+ &MP.RegisterObjectPlatformSections.Addr},
+ {*MP.DeregisterObjectPlatformSections.Name,
+ &MP.DeregisterObjectPlatformSections.Addr},
+ {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
+
+ bool RegisterMachOHeader = false;
+
+ for (auto *Sym : G.defined_symbols()) {
+ for (auto &RTSym : RuntimeSymbols) {
+ if (Sym->hasName() && Sym->getName() == RTSym.first) {
+ if (*RTSym.second)
+ return make_error<StringError>(
+ "Duplicate " + RTSym.first +
+ " detected during MachOPlatform bootstrap",
+ inconvertibleErrorCode());
+
+ if (Sym->getName() == *MP.MachOHeaderStartSymbol)
+ RegisterMachOHeader = true;
+
+ *RTSym.second = Sym->getAddress();
+ }
+ }
+ }
+
+ if (RegisterMachOHeader) {
+ // If this graph defines the macho header symbol then create the internal
+ // mapping between it and PlatformJD.
+ std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+ MP.JITDylibToHeaderAddr[&MP.PlatformJD] =
+ MP.Bootstrap.load()->MachOHeaderAddr;
+ MP.HeaderAddrToJITDylib[MP.Bootstrap.load()->MachOHeaderAddr] =
+ &MP.PlatformJD;
+ }
+
+ return Error::success();
+}
+
+Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineEnd(
+ jitlink::LinkGraph &G) {
+ std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
+ assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
+ --MP.Bootstrap.load()->ActiveGraphs;
+ // Notify Bootstrap->CV while holding the mutex because the mutex is
+ // also keeping Bootstrap->CV alive.
+ if (MP.Bootstrap.load()->ActiveGraphs == 0)
+ MP.Bootstrap.load()->CV.notify_all();
+ return Error::success();
+}
+
Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
@@ -678,6 +886,8 @@ Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
auto HeaderAddr = (*I)->getAddress();
MP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
+ // We can unconditionally add these actions to the Graph because this pass
+ // isn't used during bootstrap.
G.allocActions().push_back(
{cantFail(
WrapperFunctionCall::Create<SPSArgList<SPSString, SPSExecutorAddr>>(
@@ -859,20 +1069,7 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
}
Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
- jitlink::LinkGraph &G, JITDylib &JD) {
-
- // Add an action to register the eh-frame.
- if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
- jitlink::SectionRange R(*EHFrameSection);
- if (!R.empty())
- G.allocActions().push_back(
- {cantFail(
- WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
- MP.RegisterEHFrameSection.Addr, R.getRange())),
- cantFail(
- WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
- MP.DeregisterEHFrameSection.Addr, R.getRange()))});
- }
+ jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase) {
// Get a pointer to the thread data section if there is one. It will be used
// below.
@@ -892,22 +1089,9 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs;
- // Having merged thread BSS (if present) and thread data (if present),
- // record the resulting section range.
- if (ThreadDataSection) {
- jitlink::SectionRange R(*ThreadDataSection);
- if (!R.empty()) {
- if (MP.State != MachOPlatform::Initialized)
- return make_error<StringError>("__thread_data section encountered, but "
- "MachOPlatform has not finished booting",
- inconvertibleErrorCode());
-
- MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()});
- }
- }
-
// Collect data sections to register.
- StringRef DataSections[] = {DataDataSectionName, DataCommonSectionName};
+ StringRef DataSections[] = {DataDataSectionName, DataCommonSectionName,
+ EHFrameSectionName};
for (auto &SecName : DataSections) {
if (auto *Sec = G.findSectionByName(SecName)) {
jitlink::SectionRange R(*Sec);
@@ -916,9 +1100,16 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
}
}
+ // Having merged thread BSS (if present) and thread data (if present),
+ // record the resulting section range.
+ if (ThreadDataSection) {
+ jitlink::SectionRange R(*ThreadDataSection);
+ if (!R.empty())
+ MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()});
+ }
+
// If any platform sections were found then add an allocation action to call
// the registration function.
- bool RegistrationRequired = false;
StringRef PlatformSections[] = {
ModInitFuncSectionName, ObjCClassListSectionName,
ObjCImageInfoSectionName, ObjCSelRefsSectionName,
@@ -934,28 +1125,17 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
if (R.empty())
continue;
- RegistrationRequired = true;
MachOPlatformSecs.push_back({SecName, R.getRange()});
}
if (!MachOPlatformSecs.empty()) {
- std::optional<ExecutorAddr> HeaderAddr;
+ ExecutorAddr HeaderAddr;
{
std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
auto I = MP.JITDylibToHeaderAddr.find(&JD);
- if (I != MP.JITDylibToHeaderAddr.end())
- HeaderAddr = I->second;
- }
-
- if (!HeaderAddr) {
- // If we only found data sections and we're not done bootstrapping yet
- // then continue -- this must be a data section for the runtime itself,
- // and we don't need to register those.
- if (MP.State != MachOPlatform::Initialized && !RegistrationRequired)
- return Error::success();
-
- return make_error<StringError>("Missing header for " + JD.getName(),
- inconvertibleErrorCode());
+ assert(I != MP.JITDylibToHeaderAddr.end() &&
+ "Missing header for JITDylib");
+ HeaderAddr = I->second;
}
// Dump the scraped inits.
@@ -968,68 +1148,24 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
using SPSRegisterObjectPlatformSectionsArgs =
SPSArgList<SPSExecutorAddr,
SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>;
- G.allocActions().push_back(
+
+ shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase)
+ ? G.allocActions()
+ : MP.Bootstrap.load()->DeferredAAs;
+
+ allocActions.push_back(
{cantFail(
WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
- MP.RegisterObjectPlatformSections.Addr, *HeaderAddr,
+ MP.RegisterObjectPlatformSections.Addr, HeaderAddr,
MachOPlatformSecs)),
cantFail(
WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
- MP.DeregisterObjectPlatformSections.Addr, *HeaderAddr,
+ MP.DeregisterObjectPlatformSections.Addr, HeaderAddr,
MachOPlatformSecs))});
}
return Error::success();
}
-Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1(
- jitlink::LinkGraph &G) {
-
- // If there's no eh-frame there's nothing to do.
- auto *EHFrameSection = G.findSectionByName(EHFrameSectionName);
- if (!EHFrameSection)
- return Error::success();
-
- // If the eh-frame section is empty there's nothing to do.
- jitlink::SectionRange R(*EHFrameSection);
- if (R.empty())
- return Error::success();
-
- // Since we're linking the object containing the registration code now the
- // addresses won't be ready in the platform. We'll have to find them in this
- // graph instead.
- ExecutorAddr orc_rt_macho_register_ehframe_section;
- ExecutorAddr orc_rt_macho_deregister_ehframe_section;
- for (auto *Sym : G.defined_symbols()) {
- if (!Sym->hasName())
- continue;
- if (Sym->getName() == "___orc_rt_macho_register_ehframe_section")
- orc_rt_macho_register_ehframe_section = ExecutorAddr(Sym->getAddress());
- else if (Sym->getName() == "___orc_rt_macho_deregister_ehframe_section")
- orc_rt_macho_deregister_ehframe_section = ExecutorAddr(Sym->getAddress());
-
- if (orc_rt_macho_register_ehframe_section &&
- orc_rt_macho_deregister_ehframe_section)
- break;
- }
-
- // If we failed to find the required functions then bail out.
- if (!orc_rt_macho_register_ehframe_section ||
- !orc_rt_macho_deregister_ehframe_section)
- return make_error<StringError>("Could not find eh-frame registration "
- "functions during platform bootstrap",
- inconvertibleErrorCode());
-
- // Otherwise, add allocation actions to the graph to register eh-frames for
- // this object.
- G.allocActions().push_back(
- {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
- orc_rt_macho_register_ehframe_section, R.getRange())),
- cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
- orc_rt_macho_deregister_ehframe_section, R.getRange()))});
-
- return Error::success();
-}
-
} // End namespace orc.
} // End namespace llvm.
More information about the llvm-commits
mailing list