[compiler-rt] e256445 - [ORC-RT][ORC] Introduce ELF/*nix Platform and runtime support.
Lang Hames via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 17 22:01:37 PDT 2021
Author: Lang Hames
Date: 2021-08-18T15:00:22+10:00
New Revision: e256445bfff12013c3c4ad97da4aa69d25b175b5
URL: https://github.com/llvm/llvm-project/commit/e256445bfff12013c3c4ad97da4aa69d25b175b5
DIFF: https://github.com/llvm/llvm-project/commit/e256445bfff12013c3c4ad97da4aa69d25b175b5.diff
LOG: [ORC-RT][ORC] Introduce ELF/*nix Platform and runtime support.
This change adds support to ORCv2 and the Orc runtime library for static
initializers, C++ static destructors, and exception handler registration for
ELF-based platforms, at present Linux and FreeBSD on x86_64. It is based on the
MachO platform and runtime support introduced in bb5f97e3ad1.
Patch by Peter Housel. Thanks very much Peter!
Reviewed By: lhames
Differential Revision: https://reviews.llvm.org/D108081
Added:
compiler-rt/lib/orc/elfnix_platform.cpp
compiler-rt/lib/orc/elfnix_platform.h
compiler-rt/test/orc/TestCases/FreeBSD/lit.local.cfg.py
compiler-rt/test/orc/TestCases/FreeBSD/x86-64/lit.local.cfg.py
compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-cxa-atexit.S
compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-static-initializer.S
compiler-rt/test/orc/TestCases/Linux/lit.local.cfg.py
compiler-rt/test/orc/TestCases/Linux/x86-64/lit.local.cfg.py
compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-cxa-atexit.S
compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-static-initializer.S
llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
Modified:
compiler-rt/lib/orc/CMakeLists.txt
compiler-rt/test/orc/CMakeLists.txt
compiler-rt/test/orc/lit.cfg.py
llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
llvm/lib/ExecutionEngine/Orc/Mangling.cpp
llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
llvm/tools/llvm-jitlink/llvm-jitlink.cpp
Removed:
################################################################################
diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index 192a354f1900a..2f0e3e182a277 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -5,6 +5,7 @@ set(ORC_SOURCES
extensible_rtti.cpp
log_error_to_stderr.cpp
macho_platform.cpp
+ elfnix_platform.cpp
run_program_wrapper.cpp
)
diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp
new file mode 100644
index 0000000000000..1a0f22c635239
--- /dev/null
+++ b/compiler-rt/lib/orc/elfnix_platform.cpp
@@ -0,0 +1,454 @@
+//===- elfnix_platform.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code required to load the rest of the ELF-on-*IX runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "elfnix_platform.h"
+#include "common.h"
+#include "error.h"
+#include "wrapper_function_utils.h"
+
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <unordered_map>
+#include <vector>
+
+using namespace __orc_rt;
+using namespace __orc_rt::elfnix;
+
+// Declare function tags for functions in the JIT process.
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
+
+// eh-frame registration functions.
+// We expect these to be available for all processes.
+extern "C" void __register_frame(const void *);
+extern "C" void __deregister_frame(const void *);
+
+namespace {
+
+Error validatePointerSectionExtent(const char *SectionName,
+ const ExecutorAddressRange &SE) {
+ if (SE.size().getValue() % sizeof(uintptr_t)) {
+ std::ostringstream ErrMsg;
+ ErrMsg << std::hex << "Size of " << SectionName << " 0x"
+ << SE.StartAddress.getValue() << " -- 0x" << SE.EndAddress.getValue()
+ << " is not a pointer multiple";
+ return make_error<StringError>(ErrMsg.str());
+ }
+ return Error::success();
+}
+
+Error runInitArray(const std::vector<ExecutorAddressRange> &InitArraySections,
+ const ELFNixJITDylibInitializers &MOJDIs) {
+
+ for (const auto &ModInits : InitArraySections) {
+ if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
+ return Err;
+
+ using InitFunc = void (*)();
+ for (auto *Init : ModInits.toSpan<InitFunc>())
+ (*Init)();
+ }
+
+ return Error::success();
+}
+
+class ELFNixPlatformRuntimeState {
+private:
+ struct AtExitEntry {
+ void (*Func)(void *);
+ void *Arg;
+ };
+
+ using AtExitsVector = std::vector<AtExitEntry>;
+
+ struct PerJITDylibState {
+ void *Header = nullptr;
+ size_t RefCount = 0;
+ bool AllowReinitialization = false;
+ AtExitsVector AtExits;
+ };
+
+public:
+ static void initialize();
+ static ELFNixPlatformRuntimeState &get();
+ static void destroy();
+
+ ELFNixPlatformRuntimeState() = default;
+
+ // Delete copy and move constructors.
+ ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
+ ELFNixPlatformRuntimeState &
+ operator=(const ELFNixPlatformRuntimeState &) = delete;
+ ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
+ ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
+
+ Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
+ Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
+
+ const char *dlerror();
+ void *dlopen(string_view Name, int Mode);
+ int dlclose(void *DSOHandle);
+ void *dlsym(void *DSOHandle, string_view Symbol);
+
+ int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
+ void runAtExits(void *DSOHandle);
+
+private:
+ PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
+ PerJITDylibState *getJITDylibStateByName(string_view Path);
+ PerJITDylibState &
+ getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
+
+ Expected<ExecutorAddress> lookupSymbolInJITDylib(void *DSOHandle,
+ string_view Symbol);
+
+ Expected<ELFNixJITDylibInitializerSequence>
+ getJITDylibInitializersByName(string_view Path);
+ Expected<void *> dlopenInitialize(string_view Path, int Mode);
+ Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
+
+ static ELFNixPlatformRuntimeState *MOPS;
+
+ using InitSectionHandler =
+ Error (*)(const std::vector<ExecutorAddressRange> &Sections,
+ const ELFNixJITDylibInitializers &MOJDIs);
+ const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
+ {{".init_array", runInitArray}};
+
+ // FIXME: Move to thread-state.
+ std::string DLFcnError;
+
+ std::recursive_mutex JDStatesMutex;
+ std::unordered_map<void *, PerJITDylibState> JDStates;
+ std::unordered_map<std::string, void *> JDNameToHeader;
+};
+
+ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
+
+void ELFNixPlatformRuntimeState::initialize() {
+ assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
+ MOPS = new ELFNixPlatformRuntimeState();
+}
+
+ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
+ assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
+ return *MOPS;
+}
+
+void ELFNixPlatformRuntimeState::destroy() {
+ assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
+ delete MOPS;
+}
+
+Error ELFNixPlatformRuntimeState::registerObjectSections(
+ ELFNixPerObjectSectionsToRegister POSR) {
+ if (POSR.EHFrameSection.StartAddress)
+ __register_frame(POSR.EHFrameSection.StartAddress.toPtr<const char *>());
+
+ // TODO: Register thread data sections.
+
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::deregisterObjectSections(
+ ELFNixPerObjectSectionsToRegister POSR) {
+ if (POSR.EHFrameSection.StartAddress)
+ __deregister_frame(POSR.EHFrameSection.StartAddress.toPtr<const char *>());
+
+ return Error::success();
+}
+
+const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+
+void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+
+ // Use fast path if all JITDylibs are already loaded and don't require
+ // re-running initializers.
+ if (auto *JDS = getJITDylibStateByName(Path)) {
+ if (!JDS->AllowReinitialization) {
+ ++JDS->RefCount;
+ return JDS->Header;
+ }
+ }
+
+ auto H = dlopenInitialize(Path, Mode);
+ if (!H) {
+ DLFcnError = toString(H.takeError());
+ return nullptr;
+ }
+
+ return *H;
+}
+
+int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
+ runAtExits(DSOHandle);
+ return 0;
+}
+
+void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
+ auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
+ if (!Addr) {
+ DLFcnError = toString(Addr.takeError());
+ return 0;
+ }
+
+ return Addr->toPtr<void *>();
+}
+
+int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
+ void *DSOHandle) {
+ // FIXME: Handle out-of-memory errors, returning -1 if OOM.
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+ assert(JDS && "JITDylib state not initialized");
+ JDS->AtExits.push_back({F, Arg});
+ return 0;
+}
+
+void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
+ // FIXME: Should atexits be allowed to run concurrently with access to
+ // JDState?
+ AtExitsVector V;
+ {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+ assert(JDS && "JITDlybi state not initialized");
+ std::swap(V, JDS->AtExits);
+ }
+
+ while (!V.empty()) {
+ auto &AE = V.back();
+ AE.Func(AE.Arg);
+ V.pop_back();
+ }
+}
+
+ELFNixPlatformRuntimeState::PerJITDylibState *
+ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
+ auto I = JDStates.find(DSOHandle);
+ if (I == JDStates.end())
+ return nullptr;
+ return &I->second;
+}
+
+ELFNixPlatformRuntimeState::PerJITDylibState *
+ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
+ // FIXME: Avoid creating string copy here.
+ auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
+ if (I == JDNameToHeader.end())
+ return nullptr;
+ void *H = I->second;
+ auto J = JDStates.find(H);
+ assert(J != JDStates.end() &&
+ "JITDylib has name map entry but no header map entry");
+ return &J->second;
+}
+
+ELFNixPlatformRuntimeState::PerJITDylibState &
+ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
+ ELFNixJITDylibInitializers &MOJDIs) {
+ void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
+
+ auto &JDS = JDStates[Header];
+
+ // If this entry hasn't been created yet.
+ if (!JDS.Header) {
+ assert(!JDNameToHeader.count(MOJDIs.Name) &&
+ "JITDylib has header map entry but no name map entry");
+ JDNameToHeader[MOJDIs.Name] = Header;
+ JDS.Header = Header;
+ }
+
+ return JDS;
+}
+
+Expected<ExecutorAddress>
+ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
+ string_view Sym) {
+ Expected<ExecutorAddress> Result((ExecutorAddress()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddress>(
+ SPSExecutorAddress,
+ SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag, Result,
+ ExecutorAddress::fromPtr(DSOHandle), Sym))
+ return std::move(Err);
+ return Result;
+}
+
+Expected<ELFNixJITDylibInitializerSequence>
+ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
+ Expected<ELFNixJITDylibInitializerSequence> Result(
+ (ELFNixJITDylibInitializerSequence()));
+ std::string PathStr(Path.data(), Path.size());
+ if (auto Err =
+ WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
+ SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
+ Path))
+ return std::move(Err);
+ return Result;
+}
+
+Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
+ int Mode) {
+ // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
+ // reinitialization. We need to call in to the JIT to see if there's any new
+ // work pending.
+ auto InitSeq = getJITDylibInitializersByName(Path);
+ if (!InitSeq)
+ return InitSeq.takeError();
+
+ // Init sequences should be non-empty.
+ if (InitSeq->empty())
+ return make_error<StringError>(
+ "__orc_rt_elfnix_get_initializers returned an "
+ "empty init sequence");
+
+ // Otherwise register and run initializers for each JITDylib.
+ for (auto &MOJDIs : *InitSeq)
+ if (auto Err = initializeJITDylib(MOJDIs))
+ return std::move(Err);
+
+ // Return the header for the last item in the list.
+ auto *JDS = getJITDylibStateByHeaderAddr(
+ InitSeq->back().DSOHandleAddress.toPtr<void *>());
+ assert(JDS && "Missing state entry for JD");
+ return JDS->Header;
+}
+
+Error ELFNixPlatformRuntimeState::initializeJITDylib(
+ ELFNixJITDylibInitializers &MOJDIs) {
+
+ auto &JDS = getOrCreateJITDylibState(MOJDIs);
+ ++JDS.RefCount;
+
+ for (auto &KV : InitSections) {
+ const auto &Name = KV.first;
+ const auto &Handler = KV.second;
+ auto I = MOJDIs.InitSections.find(Name);
+ if (I != MOJDIs.InitSections.end()) {
+ if (auto Err = Handler(I->second, MOJDIs))
+ return Err;
+ }
+ }
+
+ return Error::success();
+}
+
+} // end anonymous namespace
+
+//------------------------------------------------------------------------------
+// JIT entry points
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
+ ELFNixPlatformRuntimeState::initialize();
+ return WrapperFunctionResult().release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
+ ELFNixPlatformRuntimeState::destroy();
+ return WrapperFunctionResult().release();
+}
+
+/// Wrapper function for registering metadata on a per-object basis.
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
+ handle(ArgData, ArgSize,
+ [](ELFNixPerObjectSectionsToRegister &POSR) {
+ return ELFNixPlatformRuntimeState::get().registerObjectSections(
+ std::move(POSR));
+ })
+ .release();
+}
+
+/// Wrapper for releasing per-object metadat.
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
+ handle(ArgData, ArgSize,
+ [](ELFNixPerObjectSectionsToRegister &POSR) {
+ return ELFNixPlatformRuntimeState::get()
+ .deregisterObjectSections(std::move(POSR));
+ })
+ .release();
+}
+
+//------------------------------------------------------------------------------
+// cxa_atexit support
+//------------------------------------------------------------------------------
+
+int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
+ void *dso_handle) {
+ return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
+ dso_handle);
+}
+
+void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
+ ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
+}
+
+//------------------------------------------------------------------------------
+// JIT'd dlfcn alternatives.
+//------------------------------------------------------------------------------
+
+const char *__orc_rt_elfnix_jit_dlerror() {
+ return ELFNixPlatformRuntimeState::get().dlerror();
+}
+
+void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
+ return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
+}
+
+int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
+ return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
+}
+
+void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
+ return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
+}
+
+//------------------------------------------------------------------------------
+// ELFNix Run Program
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
+ const char *JITDylibName, const char *EntrySymbolName, int argc,
+ char *argv[]) {
+ using MainTy = int (*)(int, char *[]);
+
+ void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
+ __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
+ if (!H) {
+ __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
+ return -1;
+ }
+
+ auto *Main =
+ reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
+
+ if (!Main) {
+ __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
+ return -1;
+ }
+
+ int Result = Main(argc, argv);
+
+ if (__orc_rt_elfnix_jit_dlclose(H) == -1)
+ __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
+
+ return Result;
+}
diff --git a/compiler-rt/lib/orc/elfnix_platform.h b/compiler-rt/lib/orc/elfnix_platform.h
new file mode 100644
index 0000000000000..288ed615ef923
--- /dev/null
+++ b/compiler-rt/lib/orc/elfnix_platform.h
@@ -0,0 +1,131 @@
+//===- elfnix_platform.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ORC Runtime support for dynamic loading features on ELF-based platforms.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_ELFNIX_PLATFORM_H
+#define ORC_RT_ELFNIX_PLATFORM_H
+
+#include "common.h"
+#include "executor_address.h"
+
+// Atexit functions.
+ORC_RT_INTERFACE int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
+ void *dso_handle);
+ORC_RT_INTERFACE void __orc_rt_elfnix_cxa_finalize(void *dso_handle);
+
+// dlfcn functions.
+ORC_RT_INTERFACE const char *__orc_rt_elfnix_jit_dlerror();
+ORC_RT_INTERFACE void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode);
+ORC_RT_INTERFACE int __orc_rt_elfnix_jit_dlclose(void *dso_handle);
+ORC_RT_INTERFACE void *__orc_rt_elfnix_jit_dlsym(void *dso_handle,
+ const char *symbol);
+
+namespace __orc_rt {
+namespace elfnix {
+
+struct ELFNixPerObjectSectionsToRegister {
+ ExecutorAddressRange EHFrameSection;
+ ExecutorAddressRange ThreadDataSection;
+};
+
+struct ELFNixJITDylibInitializers {
+ using SectionList = std::vector<ExecutorAddressRange>;
+
+ ELFNixJITDylibInitializers() = default;
+ ELFNixJITDylibInitializers(std::string Name, ExecutorAddress DSOHandleAddress)
+ : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {}
+
+ std::string Name;
+ ExecutorAddress DSOHandleAddress;
+
+ std::unordered_map<std::string, SectionList> InitSections;
+};
+
+class ELFNixJITDylibDeinitializers {};
+
+using ELFNixJITDylibInitializerSequence =
+ std::vector<ELFNixJITDylibInitializers>;
+
+using ELFNixJITDylibDeinitializerSequence =
+ std::vector<ELFNixJITDylibDeinitializers>;
+
+enum dlopen_mode : int {
+ ORC_RT_RTLD_LAZY = 0x1,
+ ORC_RT_RTLD_NOW = 0x2,
+ ORC_RT_RTLD_LOCAL = 0x4,
+ ORC_RT_RTLD_GLOBAL = 0x8
+};
+
+} // end namespace elfnix
+
+using SPSELFNixPerObjectSectionsToRegister =
+ SPSTuple<SPSExecutorAddressRange, SPSExecutorAddressRange>;
+
+template <>
+class SPSSerializationTraits<SPSELFNixPerObjectSectionsToRegister,
+ elfnix::ELFNixPerObjectSectionsToRegister> {
+
+public:
+ static size_t size(const elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFNixPerObjectSectionsToRegister::AsArgList::size(
+ MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool
+ serialize(SPSOutputBuffer &OB,
+ const elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFNixPerObjectSectionsToRegister::AsArgList::serialize(
+ OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFNixPerObjectSectionsToRegister::AsArgList::deserialize(
+ IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+};
+
+using SPSNamedExecutorAddressRangeSequenceMap =
+ SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>;
+
+using SPSELFNixJITDylibInitializers =
+ SPSTuple<SPSString, SPSExecutorAddress,
+ SPSNamedExecutorAddressRangeSequenceMap>;
+
+using SPSELFNixJITDylibInitializerSequence =
+ SPSSequence<SPSELFNixJITDylibInitializers>;
+
+/// Serialization traits for ELFNixJITDylibInitializers.
+template <>
+class SPSSerializationTraits<SPSELFNixJITDylibInitializers,
+ elfnix::ELFNixJITDylibInitializers> {
+public:
+ static size_t size(const elfnix::ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::size(
+ MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const elfnix::ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::serialize(
+ OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ elfnix::ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::deserialize(
+ IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+};
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_ELFNIX_PLATFORM_H
diff --git a/compiler-rt/test/orc/CMakeLists.txt b/compiler-rt/test/orc/CMakeLists.txt
index b2d31dc046915..f4aee9d7c9842 100644
--- a/compiler-rt/test/orc/CMakeLists.txt
+++ b/compiler-rt/test/orc/CMakeLists.txt
@@ -19,7 +19,7 @@ if (COMPILER_RT_BUILD_ORC AND COMPILER_RT_HAS_ORC)
endforeach()
endif()
-add_lit_testsuite(check-orc "Running the ORC runtime tests"
+add_lit_testsuite(check-orc-rt "Running the ORC runtime tests"
${ORC_TESTSUITES}
DEPENDS ${ORC_TEST_DEPS})
-set_target_properties(check-orc PROPERTIES FOLDER "Compiler-RT Misc")
+set_target_properties(check-orc-rt PROPERTIES FOLDER "Compiler-RT Misc")
diff --git a/compiler-rt/test/orc/TestCases/FreeBSD/lit.local.cfg.py b/compiler-rt/test/orc/TestCases/FreeBSD/lit.local.cfg.py
new file mode 100644
index 0000000000000..d00a28f9f961d
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/FreeBSD/lit.local.cfg.py
@@ -0,0 +1,2 @@
+if config.root.host_os != 'FreeBSD':
+ config.unsupported = True
diff --git a/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/lit.local.cfg.py b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/lit.local.cfg.py
new file mode 100644
index 0000000000000..e42e830bf013c
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/lit.local.cfg.py
@@ -0,0 +1,2 @@
+if config.root.host_arch not in ['x86_64', 'amd64']:
+ config.unsupported = True
diff --git a/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-cxa-atexit.S b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-cxa-atexit.S
new file mode 100644
index 0000000000000..7eb640642f33a
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-cxa-atexit.S
@@ -0,0 +1,39 @@
+// Test that the runtime correctly interposes ___cxa_atexit.
+//
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+
+ .text
+// OnExit destructor resets the test result override to zero.
+ .section .text._ZN6OnExitD2Ev,"axG", at progbits,_ZN6OnExitD2Ev,comdat
+ .p2align 4, 0x90
+ .type _ZN6OnExitD2Ev, at function
+_ZN6OnExitD2Ev: # @_ZN6OnExitD2Ev
+ .cfi_startproc
+ xorl %edi, %edi
+ jmp llvm_jitlink_setTestResultOverride at PLT # TAILCALL
+ .cfi_endproc
+
+// main registers the atexit and sets the test result to one.
+ .globl main
+ .p2align 4, 0x90 # -- Begin function main
+ .type main, at function
+main: # @main
+ .cfi_startproc
+# %bb.0:
+ movq _ZN6OnExitD2Ev at GOTPCREL(%rip), %rdi
+ leaq _ZL6onExit(%rip), %rsi
+ leaq __dso_handle(%rip), %rdx
+ callq __cxa_atexit at PLT
+ movl $1, %edi
+ callq llvm_jitlink_setTestResultOverride at PLT
+ xorl %eax, %eax
+ retq
+.Lfunc_end1:
+ .size main, .Lfunc_end1-main
+ .cfi_endproc
+ # -- End function
+ .type _ZL6onExit, at object # @_ZL6onExit
+ .local _ZL6onExit
+ .comm _ZL6onExit,1,1
+ .hidden __dso_handle
diff --git a/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-static-initializer.S b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-static-initializer.S
new file mode 100644
index 0000000000000..cf4d07aa5fa1e
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-static-initializer.S
@@ -0,0 +1,36 @@
+// Test that basic ELF static initializers work. The main function in this
+// test returns the value of 'x', which is initially 1 in the data section,
+// and reset to 0 if the _static_init function is run. If the static initializer
+// does not run then main will return 1, causing the test to be treated as a
+// failure.
+//
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+
+ .text
+
+ .globl main
+ .p2align 4, 0x90
+main: # @main
+ movq x at GOTPCREL(%rip), %rax
+ movl (%rax), %eax
+ retq
+
+# static initializer sets the value of 'x' to zero.
+
+ .p2align 4, 0x90
+static_init:
+ movq x at GOTPCREL(%rip), %rax
+ movl $0, (%rax)
+ retq
+
+ .data
+ .globl x
+ .p2align 2
+x:
+ .long 1
+ .size x, 4
+
+ .section .init_array,"aw", at init_array
+ .p2align 3
+ .quad static_init
diff --git a/compiler-rt/test/orc/TestCases/Linux/lit.local.cfg.py b/compiler-rt/test/orc/TestCases/Linux/lit.local.cfg.py
new file mode 100644
index 0000000000000..6eceeb1e673ef
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/lit.local.cfg.py
@@ -0,0 +1,2 @@
+if config.root.host_os != 'Linux':
+ config.unsupported = True
diff --git a/compiler-rt/test/orc/TestCases/Linux/x86-64/lit.local.cfg.py b/compiler-rt/test/orc/TestCases/Linux/x86-64/lit.local.cfg.py
new file mode 100644
index 0000000000000..1b14fb647bf6d
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/x86-64/lit.local.cfg.py
@@ -0,0 +1,2 @@
+if config.root.host_arch != 'x86_64':
+ config.unsupported = True
diff --git a/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-cxa-atexit.S b/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-cxa-atexit.S
new file mode 100644
index 0000000000000..7eb640642f33a
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-cxa-atexit.S
@@ -0,0 +1,39 @@
+// Test that the runtime correctly interposes ___cxa_atexit.
+//
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+
+ .text
+// OnExit destructor resets the test result override to zero.
+ .section .text._ZN6OnExitD2Ev,"axG", at progbits,_ZN6OnExitD2Ev,comdat
+ .p2align 4, 0x90
+ .type _ZN6OnExitD2Ev, at function
+_ZN6OnExitD2Ev: # @_ZN6OnExitD2Ev
+ .cfi_startproc
+ xorl %edi, %edi
+ jmp llvm_jitlink_setTestResultOverride at PLT # TAILCALL
+ .cfi_endproc
+
+// main registers the atexit and sets the test result to one.
+ .globl main
+ .p2align 4, 0x90 # -- Begin function main
+ .type main, at function
+main: # @main
+ .cfi_startproc
+# %bb.0:
+ movq _ZN6OnExitD2Ev at GOTPCREL(%rip), %rdi
+ leaq _ZL6onExit(%rip), %rsi
+ leaq __dso_handle(%rip), %rdx
+ callq __cxa_atexit at PLT
+ movl $1, %edi
+ callq llvm_jitlink_setTestResultOverride at PLT
+ xorl %eax, %eax
+ retq
+.Lfunc_end1:
+ .size main, .Lfunc_end1-main
+ .cfi_endproc
+ # -- End function
+ .type _ZL6onExit, at object # @_ZL6onExit
+ .local _ZL6onExit
+ .comm _ZL6onExit,1,1
+ .hidden __dso_handle
diff --git a/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-static-initializer.S b/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-static-initializer.S
new file mode 100644
index 0000000000000..cf4d07aa5fa1e
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-static-initializer.S
@@ -0,0 +1,36 @@
+// Test that basic ELF static initializers work. The main function in this
+// test returns the value of 'x', which is initially 1 in the data section,
+// and reset to 0 if the _static_init function is run. If the static initializer
+// does not run then main will return 1, causing the test to be treated as a
+// failure.
+//
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+
+ .text
+
+ .globl main
+ .p2align 4, 0x90
+main: # @main
+ movq x at GOTPCREL(%rip), %rax
+ movl (%rax), %eax
+ retq
+
+# static initializer sets the value of 'x' to zero.
+
+ .p2align 4, 0x90
+static_init:
+ movq x at GOTPCREL(%rip), %rax
+ movl $0, (%rax)
+ retq
+
+ .data
+ .globl x
+ .p2align 2
+x:
+ .long 1
+ .size x, 4
+
+ .section .init_array,"aw", at init_array
+ .p2align 3
+ .quad static_init
diff --git a/compiler-rt/test/orc/lit.cfg.py b/compiler-rt/test/orc/lit.cfg.py
index d499f26662db5..dc4bac4aed1e1 100644
--- a/compiler-rt/test/orc/lit.cfg.py
+++ b/compiler-rt/test/orc/lit.cfg.py
@@ -13,7 +13,10 @@ def build_invocation(compile_flags):
# Assume that llvm-jitlink is in the config.llvm_tools_dir.
llvm_jitlink = os.path.join(config.llvm_tools_dir, 'llvm-jitlink')
-orc_rt_path = '%s/libclang_rt.orc_osx.a' % config.compiler_rt_libdir
+if config.host_os == 'Darwin':
+ orc_rt_path = '%s/libclang_rt.orc_osx.a' % config.compiler_rt_libdir
+else:
+ orc_rt_path = '%s/libclang_rt.orc%s.a' % (config.compiler_rt_libdir, config.target_suffix)
config.substitutions.append(
('%clang ', build_invocation([config.target_cflags])))
@@ -26,5 +29,5 @@ def build_invocation(compile_flags):
# Default test suffixes.
config.suffixes = ['.c', '.cpp', '.S']
-if config.host_os not in ['Darwin']:
+if config.host_os not in ['Darwin', 'FreeBSD', 'Linux']:
config.unsupported = True
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
new file mode 100644
index 0000000000000..6e730471a70fb
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
@@ -0,0 +1,330 @@
+//===-- ELFNixPlatform.h -- Utilities for executing ELF in Orc --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Linux/BSD support for executing JIT'd ELF in Orc.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H
+#define LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h"
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+
+#include <future>
+#include <thread>
+#include <vector>
+
+namespace llvm {
+namespace orc {
+
+struct ELFPerObjectSectionsToRegister {
+ ExecutorAddressRange EHFrameSection;
+ ExecutorAddressRange ThreadDataSection;
+};
+
+struct ELFNixJITDylibInitializers {
+ using SectionList = std::vector<ExecutorAddressRange>;
+
+ ELFNixJITDylibInitializers(std::string Name, ExecutorAddress DSOHandleAddress)
+ : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {}
+
+ std::string Name;
+ ExecutorAddress DSOHandleAddress;
+
+ StringMap<SectionList> InitSections;
+};
+
+class ELFNixJITDylibDeinitializers {};
+
+using ELFNixJITDylibInitializerSequence =
+ std::vector<ELFNixJITDylibInitializers>;
+
+using ELFNixJITDylibDeinitializerSequence =
+ std::vector<ELFNixJITDylibDeinitializers>;
+
+/// Mediates between ELFNix initialization and ExecutionSession state.
+class ELFNixPlatform : public Platform {
+public:
+ /// Try to create a ELFNixPlatform instance, adding the ORC runtime to the
+ /// given JITDylib.
+ ///
+ /// The ORC runtime requires access to a number of symbols in
+ /// libc++. It is up to the caller to ensure that the requried
+ /// symbols can be referenced by code added to PlatformJD. The
+ /// standard way to achieve this is to first attach dynamic library
+ /// search generators for either the given process, or for the
+ /// specific required libraries, to PlatformJD, then to create the
+ /// platform instance:
+ ///
+ /// \code{.cpp}
+ /// auto &PlatformJD = ES.createBareJITDylib("stdlib");
+ /// PlatformJD.addGenerator(
+ /// ExitOnErr(EPCDynamicLibrarySearchGenerator
+ /// ::GetForTargetProcess(EPC)));
+ /// ES.setPlatform(
+ /// ExitOnErr(ELFNixPlatform::Create(ES, ObjLayer, EPC, PlatformJD,
+ /// "/path/to/orc/runtime")));
+ /// \endcode
+ ///
+ /// Alternatively, these symbols could be added to another JITDylib that
+ /// PlatformJD links against.
+ ///
+ /// Clients are also responsible for ensuring that any JIT'd code that
+ /// depends on runtime functions (including any code using TLV or static
+ /// destructors) can reference the runtime symbols. This is usually achieved
+ /// by linking any JITDylibs containing regular code against
+ /// PlatformJD.
+ ///
+ /// By default, ELFNixPlatform will add the set of aliases returned by the
+ /// standardPlatformAliases function. This includes both required aliases
+ /// (e.g. __cxa_atexit -> __orc_rt_elf_cxa_atexit for static destructor
+ /// support), and optional aliases that provide JIT versions of common
+ /// functions (e.g. dlopen -> __orc_rt_elf_jit_dlopen). Clients can
+ /// override these defaults by passing a non-None value for the
+ /// RuntimeAliases function, in which case the client is responsible for
+ /// setting up all aliases (including the required ones).
+ static Expected<std::unique_ptr<ELFNixPlatform>>
+ Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
+ JITDylib &PlatformJD, const char *OrcRuntimePath,
+ Optional<SymbolAliasMap> RuntimeAliases = None);
+
+ ExecutionSession &getExecutionSession() const { return ES; }
+ ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; }
+
+ Error setupJITDylib(JITDylib &JD) override;
+ Error notifyAdding(ResourceTracker &RT,
+ const MaterializationUnit &MU) override;
+ Error notifyRemoving(ResourceTracker &RT) override;
+
+ /// Returns an AliasMap containing the default aliases for the ELFNixPlatform.
+ /// This can be modified by clients when constructing the platform to add
+ /// or remove aliases.
+ static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES);
+
+ /// Returns the array of required CXX aliases.
+ static ArrayRef<std::pair<const char *, const char *>> requiredCXXAliases();
+
+ /// Returns the array of standard runtime utility aliases for ELF.
+ static ArrayRef<std::pair<const char *, const char *>>
+ standardRuntimeUtilityAliases();
+
+ /// Returns true if the given section name is an initializer section.
+ static bool isInitializerSection(StringRef SecName);
+
+private:
+ // The ELFNixPlatformPlugin scans/modifies LinkGraphs to support ELF
+ // platform features including initializers, exceptions, TLV, and language
+ // runtime registration.
+ class ELFNixPlatformPlugin : public ObjectLinkingLayer::Plugin {
+ public:
+ ELFNixPlatformPlugin(ELFNixPlatform &MP) : MP(MP) {}
+
+ void modifyPassConfig(MaterializationResponsibility &MR,
+ jitlink::LinkGraph &G,
+ jitlink::PassConfiguration &Config) override;
+
+ SyntheticSymbolDependenciesMap
+ getSyntheticSymbolDependencies(MaterializationResponsibility &MR) override;
+
+ // FIXME: We should be tentatively tracking scraped sections and discarding
+ // if the MR fails.
+ Error notifyFailed(MaterializationResponsibility &MR) override {
+ return Error::success();
+ }
+
+ Error notifyRemovingResources(ResourceKey K) override {
+ return Error::success();
+ }
+
+ void notifyTransferringResources(ResourceKey DstKey,
+ ResourceKey SrcKey) override {}
+
+ private:
+ using InitSymbolDepMap =
+ DenseMap<MaterializationResponsibility *, JITLinkSymbolSet>;
+
+ void addInitializerSupportPasses(MaterializationResponsibility &MR,
+ jitlink::PassConfiguration &Config);
+
+ void addDSOHandleSupportPasses(MaterializationResponsibility &MR,
+ jitlink::PassConfiguration &Config);
+
+ void addEHAndTLVSupportPasses(MaterializationResponsibility &MR,
+ jitlink::PassConfiguration &Config);
+
+ Error preserveInitSections(jitlink::LinkGraph &G,
+ MaterializationResponsibility &MR);
+
+ Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD);
+
+ Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD);
+
+ std::mutex PluginMutex;
+ ELFNixPlatform &MP;
+ InitSymbolDepMap InitSymbolDeps;
+ };
+
+ using SendInitializerSequenceFn =
+ unique_function<void(Expected<ELFNixJITDylibInitializerSequence>)>;
+
+ using SendDeinitializerSequenceFn =
+ unique_function<void(Expected<ELFNixJITDylibDeinitializerSequence>)>;
+
+ using SendSymbolAddressFn = unique_function<void(Expected<ExecutorAddress>)>;
+
+ static bool supportedTarget(const Triple &TT);
+
+ ELFNixPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
+ JITDylib &PlatformJD,
+ std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
+ Error &Err);
+
+ // Associate ELFNixPlatform JIT-side runtime support functions with handlers.
+ Error associateRuntimeSupportFunctions(JITDylib &PlatformJD);
+
+ void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult,
+ JITDylib &JD,
+ std::vector<JITDylibSP> DFSLinkOrder);
+
+ void getInitializersLookupPhase(SendInitializerSequenceFn SendResult,
+ JITDylib &JD);
+
+ void rt_getInitializers(SendInitializerSequenceFn SendResult,
+ StringRef JDName);
+
+ void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult,
+ ExecutorAddress Handle);
+
+ void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddress Handle,
+ StringRef SymbolName);
+
+ // Records the addresses of runtime symbols used by the platform.
+ Error bootstrapELFNixRuntime(JITDylib &PlatformJD);
+
+ Error registerInitInfo(JITDylib &JD,
+ ArrayRef<jitlink::Section *> InitSections);
+
+ Error registerPerObjectSections(const ELFPerObjectSectionsToRegister &POSR);
+
+ ExecutionSession &ES;
+ ObjectLinkingLayer &ObjLinkingLayer;
+
+ SymbolStringPtr DSOHandleSymbol;
+ std::atomic<bool> RuntimeBootstrapped{false};
+
+ ExecutorAddress orc_rt_elfnix_platform_bootstrap;
+ ExecutorAddress orc_rt_elfnix_platform_shutdown;
+ ExecutorAddress orc_rt_elfnix_register_object_sections;
+ ExecutorAddress orc_rt_elfnix_create_pthread_key;
+
+ DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols;
+
+ // InitSeqs gets its own mutex to avoid locking the whole session when
+ // aggregating data from the jitlink.
+ std::mutex PlatformMutex;
+ DenseMap<JITDylib *, ELFNixJITDylibInitializers> InitSeqs;
+ std::vector<ELFPerObjectSectionsToRegister> BootstrapPOSRs;
+
+ DenseMap<JITTargetAddress, JITDylib *> HandleAddrToJITDylib;
+ DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey;
+};
+
+namespace shared {
+
+using SPSELFPerObjectSectionsToRegister =
+ SPSTuple<SPSExecutorAddressRange, SPSExecutorAddressRange>;
+
+template <>
+class SPSSerializationTraits<SPSELFPerObjectSectionsToRegister,
+ ELFPerObjectSectionsToRegister> {
+
+public:
+ static size_t size(const ELFPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFPerObjectSectionsToRegister::AsArgList::size(
+ MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const ELFPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFPerObjectSectionsToRegister::AsArgList::serialize(
+ OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ ELFPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFPerObjectSectionsToRegister::AsArgList::deserialize(
+ IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+};
+
+using SPSNamedExecutorAddressRangeSequenceMap =
+ SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>;
+
+using SPSELFNixJITDylibInitializers =
+ SPSTuple<SPSString, SPSExecutorAddress,
+ SPSNamedExecutorAddressRangeSequenceMap>;
+
+using SPSELFNixJITDylibInitializerSequence =
+ SPSSequence<SPSELFNixJITDylibInitializers>;
+
+/// Serialization traits for ELFNixJITDylibInitializers.
+template <>
+class SPSSerializationTraits<SPSELFNixJITDylibInitializers,
+ ELFNixJITDylibInitializers> {
+public:
+ static size_t size(const ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::size(
+ MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::serialize(
+ OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::deserialize(
+ IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+};
+
+using SPSELFJITDylibDeinitializers = SPSEmpty;
+
+using SPSELFJITDylibDeinitializerSequence =
+ SPSSequence<SPSELFJITDylibDeinitializers>;
+
+template <>
+class SPSSerializationTraits<SPSELFJITDylibDeinitializers,
+ ELFNixJITDylibDeinitializers> {
+public:
+ static size_t size(const ELFNixJITDylibDeinitializers &MOJDDs) { return 0; }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const ELFNixJITDylibDeinitializers &MOJDDs) {
+ return true;
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ ELFNixJITDylibDeinitializers &MOJDDs) {
+ MOJDDs = ELFNixJITDylibDeinitializers();
+ return true;
+ }
+};
+
+} // end namespace shared
+} // end namespace orc
+} // end namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index edf9b3ff9b394..aad96ced1c2db 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -17,6 +17,7 @@ add_llvm_component_library(LLVMOrcJIT
Layer.cpp
LLJIT.cpp
MachOPlatform.cpp
+ ELFNixPlatform.cpp
Mangling.cpp
ObjectLinkingLayer.cpp
ObjectTransformLayer.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
new file mode 100644
index 0000000000000..ff8f0e90edf85
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -0,0 +1,762 @@
+//===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
+
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
+#include "llvm/ExecutionEngine/JITLink/x86_64.h"
+#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/Support/BinaryByteStream.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::orc::shared;
+
+namespace {
+
+class DSOHandleMaterializationUnit : public MaterializationUnit {
+public:
+ DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
+ const SymbolStringPtr &DSOHandleSymbol)
+ : MaterializationUnit(createDSOHandleSectionSymbols(ENP, DSOHandleSymbol),
+ DSOHandleSymbol),
+ ENP(ENP) {}
+
+ StringRef getName() const override { return "DSOHandleMU"; }
+
+ void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+ unsigned PointerSize;
+ support::endianness Endianness;
+ jitlink::Edge::Kind EdgeKind;
+ const auto &TT =
+ ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
+
+ switch (TT.getArch()) {
+ case Triple::x86_64:
+ PointerSize = 8;
+ Endianness = support::endianness::little;
+ EdgeKind = jitlink::x86_64::Pointer64;
+ break;
+ default:
+ llvm_unreachable("Unrecognized architecture");
+ }
+
+ // void *__dso_handle = &__dso_handle;
+ auto G = std::make_unique<jitlink::LinkGraph>(
+ "<DSOHandleMU>", TT, PointerSize, Endianness,
+ jitlink::getGenericEdgeKindName);
+ auto &DSOHandleSection =
+ G->createSection(".data.__dso_handle", sys::Memory::MF_READ);
+ auto &DSOHandleBlock = G->createContentBlock(
+ DSOHandleSection, getDSOHandleContent(PointerSize), 0, 8, 0);
+ auto &DSOHandleSymbol = G->addDefinedSymbol(
+ DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
+ jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
+ DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
+
+ ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
+ }
+
+ void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
+
+private:
+ static SymbolFlagsMap
+ createDSOHandleSectionSymbols(ELFNixPlatform &ENP,
+ const SymbolStringPtr &DSOHandleSymbol) {
+ SymbolFlagsMap SymbolFlags;
+ SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
+ return SymbolFlags;
+ }
+
+ ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
+ static const char Content[8] = {0};
+ assert(PointerSize <= sizeof Content);
+ return {Content, PointerSize};
+ }
+
+ ELFNixPlatform &ENP;
+};
+
+StringRef EHFrameSectionName = ".eh_frame";
+StringRef InitArrayFuncSectionName = ".init_array";
+
+StringRef ThreadBSSSectionName = ".tbss";
+StringRef ThreadDataSectionName = ".tdata";
+
+StringRef InitSectionNames[] = {InitArrayFuncSectionName};
+
+} // end anonymous namespace
+
+namespace llvm {
+namespace orc {
+
+Expected<std::unique_ptr<ELFNixPlatform>>
+ELFNixPlatform::Create(ExecutionSession &ES,
+ ObjectLinkingLayer &ObjLinkingLayer,
+ JITDylib &PlatformJD, const char *OrcRuntimePath,
+ Optional<SymbolAliasMap> RuntimeAliases) {
+
+ auto &EPC = ES.getExecutorProcessControl();
+
+ // If the target is not supported then bail out immediately.
+ if (!supportedTarget(EPC.getTargetTriple()))
+ return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
+ EPC.getTargetTriple().str(),
+ inconvertibleErrorCode());
+
+ // Create default aliases if the caller didn't supply any.
+ if (!RuntimeAliases)
+ RuntimeAliases = standardPlatformAliases(ES);
+
+ // Define the aliases.
+ if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
+ return std::move(Err);
+
+ // Add JIT-dispatch function support symbols.
+ if (auto Err = PlatformJD.define(absoluteSymbols(
+ {{ES.intern("__orc_rt_jit_dispatch"),
+ {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(),
+ JITSymbolFlags::Exported}},
+ {ES.intern("__orc_rt_jit_dispatch_ctx"),
+ {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(),
+ JITSymbolFlags::Exported}}})))
+ return std::move(Err);
+
+ // Create a generator for the ORC runtime archive.
+ auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
+ ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
+ if (!OrcRuntimeArchiveGenerator)
+ return OrcRuntimeArchiveGenerator.takeError();
+
+ // Create the instance.
+ Error Err = Error::success();
+ auto P = std::unique_ptr<ELFNixPlatform>(
+ new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD,
+ std::move(*OrcRuntimeArchiveGenerator), Err));
+ if (Err)
+ return std::move(Err);
+ return std::move(P);
+}
+
+Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
+ return JD.define(
+ std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
+ return Error::success();
+}
+
+Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
+ const MaterializationUnit &MU) {
+ auto &JD = RT.getJITDylib();
+ const auto &InitSym = MU.getInitializerSymbol();
+ if (!InitSym)
+ return Error::success();
+
+ RegisteredInitSymbols[&JD].add(InitSym,
+ SymbolLookupFlags::WeaklyReferencedSymbol);
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
+ << " for MU " << MU.getName() << "\n";
+ });
+ return Error::success();
+}
+
+Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
+ llvm_unreachable("Not supported yet");
+}
+
+static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
+ ArrayRef<std::pair<const char *, const char *>> AL) {
+ for (auto &KV : AL) {
+ auto AliasName = ES.intern(KV.first);
+ assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
+ Aliases[std::move(AliasName)] = {ES.intern(KV.second),
+ JITSymbolFlags::Exported};
+ }
+}
+
+SymbolAliasMap ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES) {
+ SymbolAliasMap Aliases;
+ addAliases(ES, Aliases, requiredCXXAliases());
+ addAliases(ES, Aliases, standardRuntimeUtilityAliases());
+ return Aliases;
+}
+
+ArrayRef<std::pair<const char *, const char *>>
+ELFNixPlatform::requiredCXXAliases() {
+ static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
+ {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"}};
+
+ return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
+}
+
+ArrayRef<std::pair<const char *, const char *>>
+ELFNixPlatform::standardRuntimeUtilityAliases() {
+ static const std::pair<const char *, const char *>
+ StandardRuntimeUtilityAliases[] = {
+ {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
+ {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
+
+ return ArrayRef<std::pair<const char *, const char *>>(
+ StandardRuntimeUtilityAliases);
+}
+
+bool ELFNixPlatform::isInitializerSection(StringRef SecName) {
+ for (auto &Name : InitSectionNames) {
+ if (Name.equals(SecName))
+ return true;
+ }
+ return false;
+}
+
+bool ELFNixPlatform::supportedTarget(const Triple &TT) {
+ switch (TT.getArch()) {
+ case Triple::x86_64:
+ return true;
+ default:
+ return false;
+ }
+}
+
+ELFNixPlatform::ELFNixPlatform(
+ ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
+ JITDylib &PlatformJD,
+ std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
+ : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
+ DSOHandleSymbol(ES.intern("__dso_handle")) {
+ ErrorAsOutParameter _(&Err);
+
+ ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
+
+ PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
+
+ // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
+ // the platform now), so set it up.
+ if (auto E2 = setupJITDylib(PlatformJD)) {
+ Err = std::move(E2);
+ return;
+ }
+
+ RegisteredInitSymbols[&PlatformJD].add(
+ DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
+
+ // Associate wrapper function tags with JIT-side function implementations.
+ if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
+ Err = std::move(E2);
+ return;
+ }
+
+ // Lookup addresses of runtime functions callable by the platform,
+ // call the platform bootstrap function to initialize the platform-state
+ // object in the executor.
+ if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
+ Err = std::move(E2);
+ return;
+ }
+}
+
+Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
+ ExecutionSession::JITDispatchHandlerAssociationMap WFs;
+
+ using GetInitializersSPSSig =
+ SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
+ WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
+ ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
+ this, &ELFNixPlatform::rt_getInitializers);
+
+ using GetDeinitializersSPSSig =
+ SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddress);
+ WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
+ ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
+ this, &ELFNixPlatform::rt_getDeinitializers);
+
+ using LookupSymbolSPSSig =
+ SPSExpected<SPSExecutorAddress>(SPSExecutorAddress, SPSString);
+ WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
+ ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
+ &ELFNixPlatform::rt_lookupSymbol);
+
+ return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
+}
+
+void ELFNixPlatform::getInitializersBuildSequencePhase(
+ SendInitializerSequenceFn SendResult, JITDylib &JD,
+ std::vector<JITDylibSP> DFSLinkOrder) {
+ ELFNixJITDylibInitializerSequence FullInitSeq;
+ {
+ std::lock_guard<std::mutex> Lock(PlatformMutex);
+ for (auto &InitJD : reverse(DFSLinkOrder)) {
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
+ << "\" to sequence\n";
+ });
+ auto ISItr = InitSeqs.find(InitJD.get());
+ if (ISItr != InitSeqs.end()) {
+ FullInitSeq.emplace_back(std::move(ISItr->second));
+ InitSeqs.erase(ISItr);
+ }
+ }
+ }
+
+ SendResult(std::move(FullInitSeq));
+}
+
+void ELFNixPlatform::getInitializersLookupPhase(
+ SendInitializerSequenceFn SendResult, JITDylib &JD) {
+
+ auto DFSLinkOrder = JD.getDFSLinkOrder();
+ DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
+ ES.runSessionLocked([&]() {
+ for (auto &InitJD : DFSLinkOrder) {
+ auto RISItr = RegisteredInitSymbols.find(InitJD.get());
+ if (RISItr != RegisteredInitSymbols.end()) {
+ NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
+ RegisteredInitSymbols.erase(RISItr);
+ }
+ }
+ });
+
+ // If there are no further init symbols to look up then move on to the next
+ // phase.
+ if (NewInitSymbols.empty()) {
+ getInitializersBuildSequencePhase(std::move(SendResult), JD,
+ std::move(DFSLinkOrder));
+ return;
+ }
+
+ // Otherwise issue a lookup and re-run this phase when it completes.
+ lookupInitSymbolsAsync(
+ [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
+ if (Err)
+ SendResult(std::move(Err));
+ else
+ getInitializersLookupPhase(std::move(SendResult), JD);
+ },
+ ES, std::move(NewInitSymbols));
+}
+
+void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
+ StringRef JDName) {
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
+ });
+
+ JITDylib *JD = ES.getJITDylibByName(JDName);
+ if (!JD) {
+ LLVM_DEBUG({
+ dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n";
+ });
+ SendResult(make_error<StringError>("No JITDylib named " + JDName,
+ inconvertibleErrorCode()));
+ return;
+ }
+
+ getInitializersLookupPhase(std::move(SendResult), *JD);
+}
+
+void ELFNixPlatform::rt_getDeinitializers(
+ SendDeinitializerSequenceFn SendResult, ExecutorAddress Handle) {
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform::rt_getDeinitializers(\""
+ << formatv("{0:x}", Handle.getValue()) << "\")\n";
+ });
+
+ JITDylib *JD = nullptr;
+
+ {
+ std::lock_guard<std::mutex> Lock(PlatformMutex);
+ auto I = HandleAddrToJITDylib.find(Handle.getValue());
+ if (I != HandleAddrToJITDylib.end())
+ JD = I->second;
+ }
+
+ if (!JD) {
+ LLVM_DEBUG({
+ dbgs() << " No JITDylib for handle "
+ << formatv("{0:x}", Handle.getValue()) << "\n";
+ });
+ SendResult(make_error<StringError>("No JITDylib associated with handle " +
+ formatv("{0:x}", Handle.getValue()),
+ inconvertibleErrorCode()));
+ return;
+ }
+
+ SendResult(ELFNixJITDylibDeinitializerSequence());
+}
+
+void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
+ ExecutorAddress Handle,
+ StringRef SymbolName) {
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform::rt_lookupSymbol(\""
+ << formatv("{0:x}", Handle.getValue()) << "\")\n";
+ });
+
+ JITDylib *JD = nullptr;
+
+ {
+ std::lock_guard<std::mutex> Lock(PlatformMutex);
+ auto I = HandleAddrToJITDylib.find(Handle.getValue());
+ if (I != HandleAddrToJITDylib.end())
+ JD = I->second;
+ }
+
+ if (!JD) {
+ LLVM_DEBUG({
+ dbgs() << " No JITDylib for handle "
+ << formatv("{0:x}", Handle.getValue()) << "\n";
+ });
+ SendResult(make_error<StringError>("No JITDylib associated with handle " +
+ formatv("{0:x}", Handle.getValue()),
+ inconvertibleErrorCode()));
+ return;
+ }
+
+ // Use functor class to work around XL build compiler issue on AIX.
+ class RtLookupNotifyComplete {
+ public:
+ RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
+ : SendResult(std::move(SendResult)) {}
+ void operator()(Expected<SymbolMap> Result) {
+ if (Result) {
+ assert(Result->size() == 1 && "Unexpected result map count");
+ SendResult(ExecutorAddress(Result->begin()->second.getAddress()));
+ } else {
+ SendResult(Result.takeError());
+ }
+ }
+
+ private:
+ SendSymbolAddressFn SendResult;
+ };
+
+ ES.lookup(
+ LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
+ SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
+ RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
+}
+
+Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
+
+ std::pair<const char *, ExecutorAddress *> Symbols[] = {
+ {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
+ {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
+ {"__orc_rt_elfnix_register_object_sections",
+ &orc_rt_elfnix_register_object_sections}};
+
+ SymbolLookupSet RuntimeSymbols;
+ std::vector<std::pair<SymbolStringPtr, ExecutorAddress *>> AddrsToRecord;
+ for (const auto &KV : Symbols) {
+ auto Name = ES.intern(KV.first);
+ RuntimeSymbols.add(Name);
+ AddrsToRecord.push_back({std::move(Name), KV.second});
+ }
+
+ auto RuntimeSymbolAddrs = ES.lookup(
+ {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
+ if (!RuntimeSymbolAddrs)
+ return RuntimeSymbolAddrs.takeError();
+
+ for (const auto &KV : AddrsToRecord) {
+ auto &Name = KV.first;
+ assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
+ KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress());
+ }
+
+ if (auto Err = ES.callSPSWrapper<void()>(
+ orc_rt_elfnix_platform_bootstrap.getValue()))
+ return Err;
+
+ // FIXME: Ordering is fuzzy here. We're probably best off saying
+ // "behavior is undefined if code that uses the runtime is added before
+ // the platform constructor returns", then move all this to the constructor.
+ RuntimeBootstrapped = true;
+ std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
+ {
+ std::lock_guard<std::mutex> Lock(PlatformMutex);
+ DeferredPOSRs = std::move(BootstrapPOSRs);
+ }
+
+ for (auto &D : DeferredPOSRs)
+ if (auto Err = registerPerObjectSections(D))
+ return Err;
+
+ return Error::success();
+}
+
+Error ELFNixPlatform::registerInitInfo(
+ JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
+
+ std::unique_lock<std::mutex> Lock(PlatformMutex);
+
+ ELFNixJITDylibInitializers *InitSeq = nullptr;
+ {
+ auto I = InitSeqs.find(&JD);
+ if (I == InitSeqs.end()) {
+ // If there's no init sequence entry yet then we need to look up the
+ // header symbol to force creation of one.
+ Lock.unlock();
+
+ auto SearchOrder =
+ JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
+ if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
+ return Err;
+
+ Lock.lock();
+ I = InitSeqs.find(&JD);
+ assert(I != InitSeqs.end() &&
+ "Entry missing after header symbol lookup?");
+ }
+ InitSeq = &I->second;
+ }
+
+ for (auto *Sec : InitSections) {
+ // FIXME: Avoid copy here.
+ jitlink::SectionRange R(*Sec);
+ InitSeq->InitSections[Sec->getName()].push_back(
+ {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())});
+ }
+
+ return Error::success();
+}
+
+Error ELFNixPlatform::registerPerObjectSections(
+ const ELFPerObjectSectionsToRegister &POSR) {
+
+ if (!orc_rt_elfnix_register_object_sections)
+ return make_error<StringError>("Attempting to register per-object "
+ "sections, but runtime support has not "
+ "been loaded yet",
+ inconvertibleErrorCode());
+
+ Error ErrResult = Error::success();
+ if (auto Err = ES.callSPSWrapper<shared::SPSError(
+ SPSELFPerObjectSectionsToRegister)>(
+ orc_rt_elfnix_register_object_sections.getValue(), ErrResult, POSR))
+ return Err;
+ return ErrResult;
+}
+
+void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
+ MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
+ jitlink::PassConfiguration &Config) {
+
+ // If the initializer symbol is the __dso_handle symbol then just add
+ // the DSO handle support passes.
+ if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
+ addDSOHandleSupportPasses(MR, Config);
+ // The DSOHandle materialization unit doesn't require any other
+ // support, so we can bail out early.
+ return;
+ }
+
+ // If the object contains initializers then add passes to record them.
+ if (MR.getInitializerSymbol())
+ addInitializerSupportPasses(MR, Config);
+
+ // Add passes for eh-frame and TLV support.
+ addEHAndTLVSupportPasses(MR, Config);
+}
+
+ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
+ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
+ MaterializationResponsibility &MR) {
+ std::lock_guard<std::mutex> Lock(PluginMutex);
+ auto I = InitSymbolDeps.find(&MR);
+ if (I != InitSymbolDeps.end()) {
+ SyntheticSymbolDependenciesMap Result;
+ Result[MR.getInitializerSymbol()] = std::move(I->second);
+ InitSymbolDeps.erase(&MR);
+ return Result;
+ }
+ return SyntheticSymbolDependenciesMap();
+}
+
+void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
+ MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
+
+ /// Preserve init sections.
+ Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
+ if (auto Err = preserveInitSections(G, MR))
+ return Err;
+ return Error::success();
+ });
+
+ Config.PostFixupPasses.push_back(
+ [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
+ return registerInitSections(G, JD);
+ });
+}
+
+void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
+ MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
+
+ Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
+ jitlink::LinkGraph &G) -> Error {
+ auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
+ return Sym->getName() == *MP.DSOHandleSymbol;
+ });
+ assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
+ {
+ std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+ JITTargetAddress HandleAddr = (*I)->getAddress();
+ MP.HandleAddrToJITDylib[HandleAddr] = &JD;
+ assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
+ MP.InitSeqs.insert(
+ std::make_pair(&JD, ELFNixJITDylibInitializers(
+ JD.getName(), ExecutorAddress(HandleAddr))));
+ }
+ return Error::success();
+ });
+}
+
+void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
+ MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
+
+ // Insert TLV lowering at the start of the PostPrunePasses, since we want
+ // it to run before GOT/PLT lowering.
+ Config.PostPrunePasses.insert(
+ Config.PostPrunePasses.begin(),
+ [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
+ return fixTLVSectionsAndEdges(G, JD);
+ });
+
+ // Add a pass to register the final addresses of the eh-frame and TLV sections
+ // with the runtime.
+ Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
+ ELFPerObjectSectionsToRegister POSR;
+
+ if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
+ jitlink::SectionRange R(*EHFrameSection);
+ if (!R.empty())
+ POSR.EHFrameSection = {ExecutorAddress(R.getStart()),
+ ExecutorAddress(R.getEnd())};
+ }
+
+ // Get a pointer to the thread data section if there is one. It will be used
+ // below.
+ jitlink::Section *ThreadDataSection =
+ G.findSectionByName(ThreadDataSectionName);
+
+ // Handle thread BSS section if there is one.
+ if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
+ // If there's already a thread data section in this graph then merge the
+ // thread BSS section content into it, otherwise just treat the thread
+ // BSS section as the thread data section.
+ if (ThreadDataSection)
+ G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
+ else
+ ThreadDataSection = ThreadBSSSection;
+ }
+
+ // Having merged thread BSS (if present) and thread data (if present),
+ // record the resulting section range.
+ if (ThreadDataSection) {
+ jitlink::SectionRange R(*ThreadDataSection);
+ if (!R.empty())
+ POSR.ThreadDataSection = {ExecutorAddress(R.getStart()),
+ ExecutorAddress(R.getEnd())};
+ }
+
+ if (POSR.EHFrameSection.StartAddress ||
+ POSR.ThreadDataSection.StartAddress) {
+
+ // If we're still bootstrapping the runtime then just record this
+ // frame for now.
+ if (!MP.RuntimeBootstrapped) {
+ std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+ MP.BootstrapPOSRs.push_back(POSR);
+ return Error::success();
+ }
+
+ // Otherwise register it immediately.
+ if (auto Err = MP.registerPerObjectSections(POSR))
+ return Err;
+ }
+
+ return Error::success();
+ });
+}
+
+Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
+ jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
+
+ JITLinkSymbolSet InitSectionSymbols;
+ for (auto &InitSectionName : InitSectionNames) {
+ // Skip non-init sections.
+ auto *InitSection = G.findSectionByName(InitSectionName);
+ if (!InitSection)
+ continue;
+
+ // Make a pass over live symbols in the section: those blocks are already
+ // preserved.
+ DenseSet<jitlink::Block *> AlreadyLiveBlocks;
+ for (auto &Sym : InitSection->symbols()) {
+ auto &B = Sym->getBlock();
+ if (Sym->isLive() && Sym->getOffset() == 0 &&
+ Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
+ InitSectionSymbols.insert(Sym);
+ AlreadyLiveBlocks.insert(&B);
+ }
+ }
+
+ // Add anonymous symbols to preserve any not-already-preserved blocks.
+ for (auto *B : InitSection->blocks())
+ if (!AlreadyLiveBlocks.count(B))
+ InitSectionSymbols.insert(
+ &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
+ }
+
+ if (!InitSectionSymbols.empty()) {
+ std::lock_guard<std::mutex> Lock(PluginMutex);
+ InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
+ }
+
+ return Error::success();
+}
+
+Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
+ jitlink::LinkGraph &G, JITDylib &JD) {
+
+ SmallVector<jitlink::Section *> InitSections;
+
+ LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; });
+
+ for (auto InitSectionName : InitSectionNames) {
+ if (auto *Sec = G.findSectionByName(InitSectionName)) {
+ InitSections.push_back(Sec);
+ }
+ }
+
+ // Dump the scraped inits.
+ LLVM_DEBUG({
+ dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
+ for (auto *Sec : InitSections) {
+ jitlink::SectionRange R(*Sec);
+ dbgs() << " " << Sec->getName() << ": "
+ << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n";
+ }
+ });
+
+ return MP.registerInitInfo(JD, InitSections);
+}
+
+Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
+ jitlink::LinkGraph &G, JITDylib &JD) {
+
+ // TODO implement TLV support
+
+ return Error::success();
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
index 14b22880ab7ec..f3377e4aebc12 100644
--- a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
@@ -7,9 +7,11 @@
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/Mangling.h"
+#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Mangler.h"
+#include "llvm/Object/ELFObjectFile.h"
#include "llvm/Object/MachO.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/Debug.h"
@@ -162,6 +164,16 @@ getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) {
break;
}
}
+ } else if (const auto *ELFObj =
+ dyn_cast<object::ELFObjectFileBase>(Obj->get())) {
+ for (auto &Sec : ELFObj->sections()) {
+ if (auto SecName = Sec.getName()) {
+ if (ELFNixPlatform::isInitializerSection(*SecName)) {
+ AddInitSymbol();
+ break;
+ }
+ }
+ }
}
return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol));
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index fd260089c04ba..3c22c93d065d8 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -64,9 +64,9 @@ class LinkGraphMaterializationUnit : public MaterializationUnit {
LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags;
}
- if (G.getTargetTriple().isOSBinFormatMachO())
- if (hasMachOInitSection(G))
- LGI.InitSymbol = makeInitSymbol(ES, G);
+ if ((G.getTargetTriple().isOSBinFormatMachO() && hasMachOInitSection(G)) ||
+ (G.getTargetTriple().isOSBinFormatELF() && hasELFInitSection(G)))
+ LGI.InitSymbol = makeInitSymbol(ES, G);
return LGI;
}
@@ -82,6 +82,13 @@ class LinkGraphMaterializationUnit : public MaterializationUnit {
return false;
}
+ static bool hasELFInitSection(LinkGraph &G) {
+ for (auto &Sec : G.sections())
+ if (Sec.getName() == ".init_array")
+ return true;
+ return false;
+ }
+
static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) {
std::string InitSymString;
raw_string_ostream(InitSymString)
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index ac2f378e8c00c..dd97bafbdfe2f 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -16,6 +16,7 @@
#include "llvm/BinaryFormat/Magic.h"
#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
+#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h"
#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
@@ -927,6 +928,14 @@ Session::Session(std::unique_ptr<ExecutorProcessControl> EPC, Error &Err)
Err = P.takeError();
return;
}
+ } else if (TT.isOSBinFormatELF() && !OrcRuntime.empty()) {
+ if (auto P =
+ ELFNixPlatform::Create(ES, ObjLayer, *MainJD, OrcRuntime.c_str()))
+ ES.setPlatform(std::move(*P));
+ else {
+ Err = P.takeError();
+ return;
+ }
} else if (!NoExec && !TT.isOSWindows() && !TT.isOSBinFormatMachO()) {
ObjLayer.addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
ES, ExitOnErr(EPCEHFrameRegistrar::Create(this->ES))));
More information about the llvm-commits
mailing list