[llvm] f2d18a4 - Reapply "[ORC] Introduce LazyReexportsManager, ... (#118923)" with fixes.
Lang Hames via llvm-commits
llvm-commits at lists.llvm.org
Sun Dec 8 19:23:00 PST 2024
Author: Lang Hames
Date: 2024-12-09T03:22:51Z
New Revision: f2d18a4d00c5f5dea436b4f7b62ec5c87b98eac2
URL: https://github.com/llvm/llvm-project/commit/f2d18a4d00c5f5dea436b4f7b62ec5c87b98eac2
DIFF: https://github.com/llvm/llvm-project/commit/f2d18a4d00c5f5dea436b4f7b62ec5c87b98eac2.diff
LOG: Reapply "[ORC] Introduce LazyReexportsManager, ... (#118923)" with fixes.
This re-applies 570ecdcf8b4, which was reverted in 74e8a37ff32 due to bot
failures. This commit renames sysv_resolve.cpp to resolve.cpp, which was the
cause of the config errors.
Added:
compiler-rt/lib/orc/resolve.cpp
compiler-rt/lib/orc/sysv_reenter.arm64.S
compiler-rt/test/orc/TestCases/Generic/Inputs/foo-ret-42.ll
compiler-rt/test/orc/TestCases/Generic/Inputs/var-x-42.ll
compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h
llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp
Modified:
compiler-rt/lib/orc/CMakeLists.txt
compiler-rt/test/orc/lit.cfg.py
llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
llvm/include/llvm/ExecutionEngine/Orc/Core.h
llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
llvm/lib/ExecutionEngine/Orc/Core.cpp
llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
llvm/tools/llvm-jitlink/llvm-jitlink.cpp
llvm/tools/llvm-jitlink/llvm-jitlink.h
Removed:
compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test
llvm/test/ExecutionEngine/JITLink/Generic/Inputs/foo-ret-42.ll
llvm/test/ExecutionEngine/JITLink/Generic/Inputs/var-x-42.ll
llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll
################################################################################
diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index 36f4349a240e34..c95700ab53876c 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -3,10 +3,11 @@
# ORC runtime library common implementation files.
set(ORC_COMMON_SOURCES
debug.cpp
+ dlfcn_wrapper.cpp
extensible_rtti.cpp
log_error_to_stderr.cpp
run_program_wrapper.cpp
- dlfcn_wrapper.cpp
+ resolve.cpp
)
# Common implementation headers will go here.
@@ -51,6 +52,7 @@ if (APPLE)
set(ORC_ASM_SOURCES
macho_tlv.x86-64.S
macho_tlv.arm64.S
+ sysv_reenter.arm64.S
)
set(ORC_IMPL_HEADERS
@@ -116,6 +118,7 @@ else() # not Apple
elfnix_tls.x86-64.S
elfnix_tls.aarch64.S
elfnix_tls.ppc64.S
+ sysv_reenter.arm64.S
)
endif()
diff --git a/compiler-rt/lib/orc/resolve.cpp b/compiler-rt/lib/orc/resolve.cpp
new file mode 100644
index 00000000000000..3d9b37d14d2a43
--- /dev/null
+++ b/compiler-rt/lib/orc/resolve.cpp
@@ -0,0 +1,49 @@
+//===- resolve.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a generic "resolver" function compatible with the
+// __orc_rt_reenter function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "executor_symbol_def.h"
+#include "jit_dispatch.h"
+#include "wrapper_function_utils.h"
+
+#include <stdio.h>
+
+#define DEBUG_TYPE "resolve"
+
+using namespace orc_rt;
+
+// Declare function tags for functions in the JIT process.
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_resolve_tag)
+
+// FIXME: Make this configurable via an alias.
+static void __orc_rt_resolve_fail(void *Caller, const char *ErrMsg) {
+ fprintf(stderr, "error resolving implementation for stub %p: %s\n", Caller,
+ ErrMsg);
+ abort();
+}
+
+extern "C" ORC_RT_HIDDEN void *__orc_rt_resolve(void *Caller) {
+ Expected<ExecutorSymbolDef> Result((ExecutorSymbolDef()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSExecutorSymbolDef>(
+ SPSExecutorAddr)>::call(JITDispatch(&__orc_rt_resolve_tag), Result,
+ ExecutorAddr::fromPtr(Caller))) {
+ __orc_rt_resolve_fail(Caller, toString(std::move(Err)).c_str());
+ return nullptr; // Unreachable.
+ }
+
+ if (!Result) {
+ __orc_rt_resolve_fail(Caller, toString(Result.takeError()).c_str());
+ return nullptr; // Unreachable.
+ }
+
+ return Result->getAddress().toPtr<void *>();
+}
diff --git a/compiler-rt/lib/orc/sysv_reenter.arm64.S b/compiler-rt/lib/orc/sysv_reenter.arm64.S
new file mode 100644
index 00000000000000..bd74a33bc63446
--- /dev/null
+++ b/compiler-rt/lib/orc/sysv_reenter.arm64.S
@@ -0,0 +1,102 @@
+//===-- sysv_reenter.arm64.s ------------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+// The content of this file is arm64-only
+#if defined(__arm64__) || defined(__aarch64__)
+
+ .text
+
+ // Saves GPRs, calls __orc_rt_sysv_resolve
+ .globl __orc_rt_sysv_reenter
+__orc_rt_sysv_reenter:
+ // Save register state, set up new stack frome.
+ stp x27, x28, [sp, #-16]!
+ stp x25, x26, [sp, #-16]!
+ stp x23, x24, [sp, #-16]!
+ stp x21, x22, [sp, #-16]!
+ stp x19, x20, [sp, #-16]!
+ stp x14, x15, [sp, #-16]!
+ stp x12, x13, [sp, #-16]!
+ stp x10, x11, [sp, #-16]!
+ stp x8, x9, [sp, #-16]!
+ stp x6, x7, [sp, #-16]!
+ stp x4, x5, [sp, #-16]!
+ stp x2, x3, [sp, #-16]!
+ stp x0, x1, [sp, #-16]!
+ stp q30, q31, [sp, #-32]!
+ stp q28, q29, [sp, #-32]!
+ stp q26, q27, [sp, #-32]!
+ stp q24, q25, [sp, #-32]!
+ stp q22, q23, [sp, #-32]!
+ stp q20, q21, [sp, #-32]!
+ stp q18, q19, [sp, #-32]!
+ stp q16, q17, [sp, #-32]!
+ stp q14, q15, [sp, #-32]!
+ stp q12, q13, [sp, #-32]!
+ stp q10, q11, [sp, #-32]!
+ stp q8, q9, [sp, #-32]!
+ stp q6, q7, [sp, #-32]!
+ stp q4, q5, [sp, #-32]!
+ stp q2, q3, [sp, #-32]!
+ stp q0, q1, [sp, #-32]!
+
+ // Look up the return address and subtract 8 from it (on the
+ // assumption that it's a standard arm64 reentry trampoline) to get
+ // back the trampoline's address.
+ sub x0, x30, #8
+
+ // Call __orc_rt_sysv_resolve to look up the implementation
+ // corresponding to the calling stub, then store this in x17 (which
+ // we'll return to below.
+#if !defined(__APPLE__)
+ bl __orc_rt_resolve
+#else
+ bl ___orc_rt_resolve
+#endif
+ mov x17, x0
+
+ // Restore the register state.
+ ldp q0, q1, [sp], #32
+ ldp q2, q3, [sp], #32
+ ldp q4, q5, [sp], #32
+ ldp q6, q7, [sp], #32
+ ldp q8, q9, [sp], #32
+ ldp q10, q11, [sp], #32
+ ldp q12, q13, [sp], #32
+ ldp q14, q15, [sp], #32
+ ldp q16, q17, [sp], #32
+ ldp q18, q19, [sp], #32
+ ldp q20, q21, [sp], #32
+ ldp q22, q23, [sp], #32
+ ldp q24, q25, [sp], #32
+ ldp q26, q27, [sp], #32
+ ldp q28, q29, [sp], #32
+ ldp q30, q31, [sp], #32
+ ldp x0, x1, [sp], #16
+ ldp x2, x3, [sp], #16
+ ldp x4, x5, [sp], #16
+ ldp x6, x7, [sp], #16
+ ldp x8, x9, [sp], #16
+ ldp x10, x11, [sp], #16
+ ldp x12, x13, [sp], #16
+ ldp x14, x15, [sp], #16
+ ldp x19, x20, [sp], #16
+ ldp x21, x22, [sp], #16
+ ldp x23, x24, [sp], #16
+ ldp x25, x26, [sp], #16
+ ldp x27, x28, [sp], #16
+ ldp x29, x30, [sp], #16
+
+ // Return to the function implementation (rather than the stub).
+ ret x17
+
+#endif // defined(__arm64__) || defined(__aarch64__)
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/Inputs/foo-ret-42.ll b/compiler-rt/test/orc/TestCases/Generic/Inputs/foo-ret-42.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/JITLink/Generic/Inputs/foo-ret-42.ll
rename to compiler-rt/test/orc/TestCases/Generic/Inputs/foo-ret-42.ll
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/Inputs/var-x-42.ll b/compiler-rt/test/orc/TestCases/Generic/Inputs/var-x-42.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/JITLink/Generic/Inputs/var-x-42.ll
rename to compiler-rt/test/orc/TestCases/Generic/Inputs/var-x-42.ll
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll b/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
similarity index 70%
rename from llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll
rename to compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
index 72325e198721b6..e722b813c3fe06 100644
--- a/llvm/test/ExecutionEngine/JITLink/Generic/lazy-link.ll
+++ b/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
@@ -4,13 +4,14 @@
; referenced by main, should be linked (despite being passed with -lazy).
;
; RUN: rm -rf %t && mkdir -p %t
-; RUN: llc -filetype=obj -o %t/foo.o %S/Inputs/foo-ret-42.ll
-; RUN: llc -filetype=obj -o %t/x.o %S/Inputs/var-x-42.ll
-; RUN: llc -filetype=obj -o %t/main.o %s
-; RUN: llvm-jitlink -noexec -show-linked-files %t/main.o -lazy %t/foo.o \
+; RUN: %clang -c -o %t/foo.o %S/Inputs/foo-ret-42.ll
+; RUN: %clang -c -o %t/x.o %S/Inputs/var-x-42.ll
+; RUN: %clang -c -o %t/main.o %s
+; RUN: %llvm_jitlink -noexec -show-linked-files %t/main.o -lazy %t/foo.o \
; RUN: -lazy %t/x.o | FileCheck %s
;
-; UNSUPPORTED: system-windows, target={{arm[^6][^4].*}}, target=powerpc64{{.*}}
+; UNSUPPORTED: system-windows
+; REQUIRES: target={{(arm|aarch)64.*}}
;
; CHECK: Linking {{.*}}main.o
; CHECK-DAG: Linking <indirect stubs graph #1>
diff --git a/compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test b/compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test
deleted file mode 100644
index 951688d7961d4b..00000000000000
--- a/compiler-rt/test/orc/TestCases/Generic/orc-rt-executor-usage.test
+++ /dev/null
@@ -1,6 +0,0 @@
-// Test that the orc-remote-executor tool errors out as expected when called
-// with no arguments.
-//
-// RUN: not %orc_rt_executor 2>&1 | FileCheck %s
-
-// CHECK: usage: orc-rt-executor [help] [<mode>] <program arguments>...
diff --git a/compiler-rt/test/orc/lit.cfg.py b/compiler-rt/test/orc/lit.cfg.py
index 6dfa94b11cc9d9..7a6eb4e7de3256 100644
--- a/compiler-rt/test/orc/lit.cfg.py
+++ b/compiler-rt/test/orc/lit.cfg.py
@@ -16,6 +16,11 @@
host_arch_compatible = True
if host_arch_compatible:
config.available_features.add("host-arch-compatible")
+
+# If the target OS hasn't been set then assume host.
+if not config.target_os:
+ config.target_os = config.host_os
+
config.test_target_is_host_executable = (
config.target_os == config.host_os and host_arch_compatible
)
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
index db440c378d24ff..1fa957178a122e 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
@@ -755,6 +755,32 @@ inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G,
sizeof(PointerJumpStubContent), true, false);
}
+/// AArch64 reentry trampoline.
+///
+/// Contains the instruction sequence for a trampoline that stores its return
+/// address on the stack and passes its own address in x0:
+/// STP x29, x30, [sp, #-16]!
+/// BL <reentry-symbol>
+extern const char ReentryTrampolineContent[8];
+
+/// Create a block of N reentry trampolines.
+inline Block &createReentryTrampolineBlock(LinkGraph &G,
+ Section &TrampolineSection,
+ Symbol &ReentrySymbol) {
+ auto &B = G.createContentBlock(TrampolineSection, ReentryTrampolineContent,
+ orc::ExecutorAddr(~uint64_t(7)), 4, 0);
+ B.addEdge(Branch26PCRel, 4, ReentrySymbol, 0);
+ return B;
+}
+
+inline Symbol &createAnonymousReentryTrampoline(LinkGraph &G,
+ Section &TrampolineSection,
+ Symbol &ReentrySymbol) {
+ return G.addAnonymousSymbol(
+ createReentryTrampolineBlock(G, TrampolineSection, ReentrySymbol), 0,
+ sizeof(ReentryTrampolineContent), true, false);
+}
+
/// Global Offset Table Builder.
class GOTTableManager : public TableManager<GOTTableManager> {
public:
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index e892005c53d8ec..9e3e46285c4085 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -51,6 +51,26 @@ enum class SymbolState : uint8_t;
using ResourceTrackerSP = IntrusiveRefCntPtr<ResourceTracker>;
using JITDylibSP = IntrusiveRefCntPtr<JITDylib>;
+/// A definition of a Symbol within a JITDylib.
+class SymbolInstance {
+public:
+ using LookupAsyncOnCompleteFn =
+ unique_function<void(Expected<ExecutorSymbolDef>)>;
+
+ SymbolInstance(JITDylibSP JD, SymbolStringPtr Name)
+ : JD(std::move(JD)), Name(std::move(Name)) {}
+
+ const JITDylib &getJITDylib() const { return *JD; }
+ const SymbolStringPtr &getName() const { return Name; }
+
+ Expected<ExecutorSymbolDef> lookup() const;
+ void lookupAsync(LookupAsyncOnCompleteFn OnComplete) const;
+
+private:
+ JITDylibSP JD;
+ SymbolStringPtr Name;
+};
+
using ResourceKey = uintptr_t;
/// API to remove / transfer ownership of JIT resources.
@@ -550,6 +570,9 @@ class MaterializationResponsibility {
/// emitted or notified of an error.
~MaterializationResponsibility();
+ /// Return the ResourceTracker associated with this instance.
+ const ResourceTrackerSP &getResourceTracker() const { return RT; }
+
/// Runs the given callback under the session lock, passing in the associated
/// ResourceKey. This is the safe way to associate resources with trackers.
template <typename Func> Error withResourceKeyDo(Func &&F) const {
@@ -1748,6 +1771,10 @@ class ExecutionSession {
JITDispatchHandlers;
};
+inline Expected<ExecutorSymbolDef> SymbolInstance::lookup() const {
+ return JD->getExecutionSession().lookup({JD.get()}, Name);
+}
+
template <typename Func> Error ResourceTracker::withResourceKeyDo(Func &&F) {
return getJITDylib().getExecutionSession().runSessionLocked([&]() -> Error {
if (isDefunct())
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
index 54442c91096b39..3da5e90a0ec5bc 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
@@ -138,6 +138,11 @@ class ELFNixPlatform : public Platform {
static ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases();
+ /// Returns a list of aliases required to enable lazy compilation via the
+ /// ORC runtime.
+ static ArrayRef<std::pair<const char *, const char *>>
+ standardLazyCompilationAliases();
+
private:
// Data needed for bootstrap only.
struct BootstrapInfo {
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h
new file mode 100644
index 00000000000000..19075c76a6073a
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkLazyCallThroughManager.h
@@ -0,0 +1,26 @@
+//===- JITLinkLazyCallThroughManager.h - JITLink based laziness -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Redirectable Symbol Manager implementation using JITLink
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKLAZYCALLTHROUGHMANAGER_H
+#define LLVM_EXECUTIONENGINE_ORC_JITLINKLAZYCALLTHROUGHMANAGER_H
+
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
+#include "llvm/Support/StringSaver.h"
+
+#include <atomic>
+
+namespace llvm {
+namespace orc {} // namespace orc
+} // namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_JITLINKLAZYCALLTHROUGHMANAGER_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
index 81d1d154d56008..83339e56cfa5f7 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h
@@ -39,12 +39,6 @@ class JITLinkRedirectableSymbolManager : public RedirectableSymbolManager {
ObjLinkingLayer, AnonymousPtrCreator, PtrJumpStubCreator));
}
- void emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> R,
- SymbolMap InitialDests) override;
-
- Error redirect(JITDylib &JD, const SymbolMap &NewDests) override;
-
-private:
JITLinkRedirectableSymbolManager(
ObjectLinkingLayer &ObjLinkingLayer,
jitlink::AnonymousPointerCreator &AnonymousPtrCreator,
@@ -53,6 +47,14 @@ class JITLinkRedirectableSymbolManager : public RedirectableSymbolManager {
AnonymousPtrCreator(std::move(AnonymousPtrCreator)),
PtrJumpStubCreator(std::move(PtrJumpStubCreator)) {}
+ ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; }
+
+ void emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> R,
+ SymbolMap InitialDests) override;
+
+ Error redirect(JITDylib &JD, const SymbolMap &NewDests) override;
+
+private:
ObjectLinkingLayer &ObjLinkingLayer;
jitlink::AnonymousPointerCreator AnonymousPtrCreator;
jitlink::PointerJumpStubCreator PtrJumpStubCreator;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
new file mode 100644
index 00000000000000..673019b748b3b2
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h
@@ -0,0 +1,72 @@
+//===- JITLinkReentryTrampolines.h -- JITLink-based trampolines -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Emit reentry trampolines via JITLink.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
+#define LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
+
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm::jitlink {
+class Block;
+class LinkGraph;
+class Section;
+class Symbol;
+} // namespace llvm::jitlink
+
+namespace llvm::orc {
+
+class ObjectLinkingLayer;
+class RedirectableSymbolManager;
+
+/// Produces trampolines on request using JITLink.
+class JITLinkReentryTrampolines {
+public:
+ using EmitTrampolineFn = unique_function<jitlink::Symbol &(
+ jitlink::LinkGraph &G, jitlink::Section &Sec,
+ jitlink::Symbol &ReentrySym)>;
+ using OnTrampolinesReadyFn = unique_function<void(
+ Expected<std::vector<ExecutorSymbolDef>> EntryAddrs)>;
+
+ /// Create trampolines using the default reentry trampoline function for
+ /// the session triple.
+ static Expected<std::unique_ptr<JITLinkReentryTrampolines>>
+ Create(ObjectLinkingLayer &ObjLinkingLayer);
+
+ JITLinkReentryTrampolines(ObjectLinkingLayer &ObjLinkingLayer,
+ EmitTrampolineFn EmitTrampoline);
+ JITLinkReentryTrampolines(JITLinkReentryTrampolines &&) = delete;
+ JITLinkReentryTrampolines &operator=(JITLinkReentryTrampolines &&) = delete;
+
+ void emit(ResourceTrackerSP RT, size_t NumTrampolines,
+ OnTrampolinesReadyFn OnTrampolinesReady);
+
+private:
+ class TrampolineAddrScraperPlugin;
+
+ ObjectLinkingLayer &ObjLinkingLayer;
+ TrampolineAddrScraperPlugin *TrampolineAddrScraper = nullptr;
+ EmitTrampolineFn EmitTrampoline;
+ std::atomic<size_t> ReentryGraphIdx{0};
+};
+
+Expected<std::unique_ptr<LazyReexportsManager>>
+createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
+ RedirectableSymbolManager &RSMgr,
+ JITDylib &PlatformJD);
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_JITLINKREENTRYTRAMPOLINES_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
index 800f6773f16f55..96223d71e4f6ae 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h
@@ -18,14 +18,13 @@
namespace llvm::orc {
class ObjectLinkingLayer;
-class LazyCallThroughManager;
+class LazyReexportsManager;
class RedirectableSymbolManager;
class LazyObjectLinkingLayer : public ObjectLayer {
public:
LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
- LazyCallThroughManager &LCTMgr,
- RedirectableSymbolManager &RSMgr);
+ LazyReexportsManager &LRMgr);
llvm::Error add(llvm::orc::ResourceTrackerSP RT,
std::unique_ptr<llvm::MemoryBuffer> O,
@@ -38,8 +37,7 @@ class LazyObjectLinkingLayer : public ObjectLayer {
class RenamerPlugin;
ObjectLinkingLayer &BaseLayer;
- LazyCallThroughManager &LCTMgr;
- RedirectableSymbolManager &RSMgr;
+ LazyReexportsManager &LRMgr;
};
} // namespace llvm::orc
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
index 6a43cb6fb6ca97..0dcf646b12dd81 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
@@ -173,6 +173,70 @@ lazyReexports(LazyCallThroughManager &LCTManager,
LCTManager, RSManager, SourceJD, std::move(CallableAliases), SrcJDLoc);
}
+class LazyReexportsManager {
+
+ friend std::unique_ptr<MaterializationUnit>
+ lazyReexports(LazyReexportsManager &, SymbolAliasMap);
+
+public:
+ using OnTrampolinesReadyFn = unique_function<void(
+ Expected<std::vector<ExecutorSymbolDef>> EntryAddrs)>;
+ using EmitTrampolinesFn =
+ unique_function<void(ResourceTrackerSP RT, size_t NumTrampolines,
+ OnTrampolinesReadyFn OnTrampolinesReady)>;
+
+ /// Create a LazyReexportsManager that uses the ORC runtime for reentry.
+ /// This will work both in-process and out-of-process.
+ static Expected<std::unique_ptr<LazyReexportsManager>>
+ Create(EmitTrampolinesFn EmitTrampolines, RedirectableSymbolManager &RSMgr,
+ JITDylib &PlatformJD);
+
+ LazyReexportsManager(LazyReexportsManager &&) = delete;
+ LazyReexportsManager &operator=(LazyReexportsManager &&) = delete;
+
+private:
+ struct CallThroughInfo {
+ SymbolStringPtr Name;
+ SymbolStringPtr BodyName;
+ JITDylibSP JD;
+ };
+
+ class MU;
+ class Plugin;
+
+ using ResolveSendResultFn =
+ unique_function<void(Expected<ExecutorSymbolDef>)>;
+
+ LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
+ RedirectableSymbolManager &RSMgr, JITDylib &PlatformJD,
+ Error &Err);
+
+ std::unique_ptr<MaterializationUnit>
+ createLazyReexports(SymbolAliasMap Reexports);
+
+ void emitReentryTrampolines(std::unique_ptr<MaterializationResponsibility> MR,
+ SymbolAliasMap Reexports);
+ void emitRedirectableSymbols(
+ std::unique_ptr<MaterializationResponsibility> MR,
+ SymbolAliasMap Reexports,
+ Expected<std::vector<ExecutorSymbolDef>> ReentryPoints);
+ void resolve(ResolveSendResultFn SendResult, ExecutorAddr ReentryStubAddr);
+
+ EmitTrampolinesFn EmitTrampolines;
+ RedirectableSymbolManager &RSMgr;
+
+ std::mutex M;
+ DenseMap<ExecutorAddr, CallThroughInfo> CallThroughs;
+};
+
+/// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export
+/// is a callable symbol that will look up and dispatch to the given aliasee on
+/// first call. All subsequent calls will go directly to the aliasee.
+inline std::unique_ptr<MaterializationUnit>
+lazyReexports(LazyReexportsManager &LRM, SymbolAliasMap Reexports) {
+ return LRM.createLazyReexports(std::move(Reexports));
+}
+
} // End namespace orc
} // End namespace llvm
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
index 19f935d6658234..1f11d9f61f6a19 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h
@@ -176,6 +176,11 @@ class MachOPlatform : public Platform {
static ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases();
+ /// Returns a list of aliases required to enable lazy compilation via the
+ /// ORC runtime.
+ static ArrayRef<std::pair<const char *, const char *>>
+ standardLazyCompilationAliases();
+
private:
using SymbolTableVector = SmallVector<
std::tuple<ExecutorAddr, ExecutorAddr, MachOExecutorSymbolFlags>>;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
index a1a5ffcf340669..f3d4c769387785 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RedirectionManager.h
@@ -23,6 +23,7 @@ namespace orc {
class RedirectionManager {
public:
virtual ~RedirectionManager() = default;
+
/// Change the redirection destination of given symbols to new destination
/// symbols.
virtual Error redirect(JITDylib &JD, const SymbolMap &NewDests) = 0;
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index a79dbd5e4494f4..968ed217d8a963 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -29,6 +29,11 @@ const char PointerJumpStubContent[12] = {
0x00, 0x02, 0x1f, (char)0xd6u // BR x16
};
+const char ReentryTrampolineContent[8] = {
+ (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
+ 0x00, 0x00, 0x00, (char)0x94 // BL
+};
+
const char *getEdgeKindName(Edge::Kind R) {
switch (R) {
case Pointer64:
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index ebfbeb990180fd..5615ad94006d3b 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -29,6 +29,7 @@ add_llvm_component_library(LLVMOrcJIT
IRTransformLayer.cpp
IRPartitionLayer.cpp
JITTargetMachineBuilder.cpp
+ JITLinkReentryTrampolines.cpp
LazyObjectLinkingLayer.cpp
LazyReexports.cpp
Layer.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index 3547eabdd0ae73..6a9ebb41e79f6b 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -178,6 +178,27 @@ void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const {
<< ": " << Symbols;
}
+void SymbolInstance::lookupAsync(LookupAsyncOnCompleteFn OnComplete) const {
+ JD->getExecutionSession().lookup(
+ LookupKind::Static, {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
+ SymbolLookupSet(Name), SymbolState::Ready,
+ [OnComplete = std::move(OnComplete)
+#ifndef NDEBUG
+ ,
+ Name = this->Name // Captured for the assert below only.
+#endif // NDEBUG
+ ](Expected<SymbolMap> Result) mutable {
+ if (Result) {
+ assert(Result->size() == 1 && "Unexpected number of results");
+ assert(Result->count(Name) &&
+ "Result does not contain expected symbol");
+ OnComplete(Result->begin()->second);
+ } else
+ OnComplete(Result.takeError());
+ },
+ NoDependenciesToRegister);
+}
+
AsynchronousSymbolQuery::AsynchronousSymbolQuery(
const SymbolLookupSet &Symbols, SymbolState RequiredState,
SymbolsResolvedCallback NotifyComplete)
@@ -1455,10 +1476,9 @@ void JITDylib::installMaterializationUnit(
void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
const SymbolNameSet &QuerySymbols) {
for (auto &QuerySymbol : QuerySymbols) {
- assert(MaterializingInfos.count(QuerySymbol) &&
- "QuerySymbol does not have MaterializingInfo");
- auto &MI = MaterializingInfos[QuerySymbol];
- MI.removeQuery(Q);
+ auto MII = MaterializingInfos.find(QuerySymbol);
+ if (MII != MaterializingInfos.end())
+ MII->second.removeQuery(Q);
}
}
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index 12859d8f7d22eb..5f017d9979c3c3 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -349,6 +349,7 @@ ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
SymbolAliasMap Aliases;
addAliases(ES, Aliases, requiredCXXAliases());
addAliases(ES, Aliases, standardRuntimeUtilityAliases());
+ addAliases(ES, Aliases, standardLazyCompilationAliases());
return Aliases;
}
@@ -377,6 +378,16 @@ ELFNixPlatform::standardRuntimeUtilityAliases() {
StandardRuntimeUtilityAliases);
}
+ArrayRef<std::pair<const char *, const char *>>
+ELFNixPlatform::standardLazyCompilationAliases() {
+ static const std::pair<const char *, const char *>
+ StandardLazyCompilationAliases[] = {
+ {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
+
+ return ArrayRef<std::pair<const char *, const char *>>(
+ StandardLazyCompilationAliases);
+}
+
bool ELFNixPlatform::supportedTarget(const Triple &TT) {
switch (TT.getArch()) {
case Triple::x86_64:
diff --git a/llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp b/llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp
new file mode 100644
index 00000000000000..1f6ec783b6c1d7
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/JITLinkReentryTrampolines.cpp
@@ -0,0 +1,184 @@
+//===----- JITLinkReentryTrampolines.cpp -- JITLink-based trampoline- -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"
+
+#include "llvm/ExecutionEngine/JITLink/aarch64.h"
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
+
+#include <memory>
+
+#define DEBUG_TYPE "orc"
+
+using namespace llvm;
+using namespace llvm::jitlink;
+
+namespace {
+constexpr StringRef ReentryFnName = "__orc_rt_reenter";
+constexpr StringRef ReentrySectionName = "__orc_stubs";
+} // namespace
+
+namespace llvm::orc {
+
+class JITLinkReentryTrampolines::TrampolineAddrScraperPlugin
+ : public ObjectLinkingLayer::Plugin {
+public:
+ void modifyPassConfig(MaterializationResponsibility &MR,
+ jitlink::LinkGraph &G,
+ jitlink::PassConfiguration &Config) override {
+ Config.PreFixupPasses.push_back(
+ [this](LinkGraph &G) { return recordTrampolineAddrs(G); });
+ }
+
+ Error notifyFailed(MaterializationResponsibility &MR) override {
+ return Error::success();
+ }
+
+ Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
+ return Error::success();
+ }
+
+ void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
+ ResourceKey SrcKey) override {}
+
+ void registerGraph(LinkGraph &G,
+ std::shared_ptr<std::vector<ExecutorSymbolDef>> Addrs) {
+ std::lock_guard<std::mutex> Lock(M);
+ assert(!PendingAddrs.count(&G) && "Duplicate registration");
+ PendingAddrs[&G] = std::move(Addrs);
+ }
+
+ Error recordTrampolineAddrs(LinkGraph &G) {
+ std::shared_ptr<std::vector<ExecutorSymbolDef>> Addrs;
+ {
+ std::lock_guard<std::mutex> Lock(M);
+ auto I = PendingAddrs.find(&G);
+ if (I == PendingAddrs.end())
+ return Error::success();
+ Addrs = std::move(I->second);
+ PendingAddrs.erase(I);
+ }
+
+ auto *Sec = G.findSectionByName(ReentrySectionName);
+ assert(Sec && "Reentry graph missing reentry section");
+ assert(!Sec->empty() && "Reentry graph is empty");
+
+ for (auto *Sym : Sec->symbols())
+ if (!Sym->hasName())
+ Addrs->push_back({Sym->getAddress(), JITSymbolFlags()});
+
+ return Error::success();
+ }
+
+private:
+ std::mutex M;
+ DenseMap<LinkGraph *, std::shared_ptr<std::vector<ExecutorSymbolDef>>>
+ PendingAddrs;
+};
+
+Expected<std::unique_ptr<JITLinkReentryTrampolines>>
+JITLinkReentryTrampolines::Create(ObjectLinkingLayer &ObjLinkingLayer) {
+
+ EmitTrampolineFn EmitTrampoline;
+
+ switch (ObjLinkingLayer.getExecutionSession().getTargetTriple().getArch()) {
+ case Triple::aarch64:
+ EmitTrampoline = aarch64::createAnonymousReentryTrampoline;
+ break;
+ default:
+ return make_error<StringError>("Architecture not supported",
+ inconvertibleErrorCode());
+ }
+
+ return std::make_unique<JITLinkReentryTrampolines>(ObjLinkingLayer,
+ std::move(EmitTrampoline));
+}
+
+JITLinkReentryTrampolines::JITLinkReentryTrampolines(
+ ObjectLinkingLayer &ObjLinkingLayer, EmitTrampolineFn EmitTrampoline)
+ : ObjLinkingLayer(ObjLinkingLayer),
+ EmitTrampoline(std::move(EmitTrampoline)) {
+ auto TAS = std::make_shared<TrampolineAddrScraperPlugin>();
+ TrampolineAddrScraper = TAS.get();
+ ObjLinkingLayer.addPlugin(std::move(TAS));
+}
+
+void JITLinkReentryTrampolines::emit(ResourceTrackerSP RT,
+ size_t NumTrampolines,
+ OnTrampolinesReadyFn OnTrampolinesReady) {
+
+ if (NumTrampolines == 0)
+ return OnTrampolinesReady(std::vector<ExecutorSymbolDef>());
+
+ JITDylibSP JD(&RT->getJITDylib());
+ auto &ES = ObjLinkingLayer.getExecutionSession();
+ Triple TT = ES.getTargetTriple();
+
+ auto ReentryGraphSym =
+ ES.intern(("__orc_reentry_graph_#" + Twine(++ReentryGraphIdx)).str());
+
+ auto G = std::make_unique<jitlink::LinkGraph>(
+ (*ReentryGraphSym).str(), ES.getSymbolStringPool(), TT,
+ TT.isArch64Bit() ? 8 : 4,
+ TT.isLittleEndian() ? endianness::little : endianness::big,
+ jitlink::getGenericEdgeKindName);
+
+ auto &ReentryFnSym = G->addExternalSymbol(ReentryFnName, 0, false);
+
+ auto &ReentrySection =
+ G->createSection(ReentrySectionName, MemProt::Exec | MemProt::Read);
+
+ for (size_t I = 0; I != NumTrampolines; ++I)
+ EmitTrampoline(*G, ReentrySection, ReentryFnSym).setLive(true);
+
+ auto &FirstBlock = **ReentrySection.blocks().begin();
+ G->addDefinedSymbol(FirstBlock, 0, *ReentryGraphSym, FirstBlock.getSize(),
+ Linkage::Strong, Scope::SideEffectsOnly, true, true);
+
+ auto TrampolineAddrs = std::make_shared<std::vector<ExecutorSymbolDef>>();
+ TrampolineAddrScraper->registerGraph(*G, TrampolineAddrs);
+
+ // Add Graph via object linking layer.
+ if (auto Err = ObjLinkingLayer.add(std::move(RT), std::move(G)))
+ return OnTrampolinesReady(std::move(Err));
+
+ // Trigger graph emission.
+ ES.lookup(
+ LookupKind::Static, {{JD.get(), JITDylibLookupFlags::MatchAllSymbols}},
+ SymbolLookupSet(ReentryGraphSym,
+ SymbolLookupFlags::WeaklyReferencedSymbol),
+ SymbolState::Ready,
+ [OnTrampolinesReady = std::move(OnTrampolinesReady),
+ TrampolineAddrs =
+ std::move(TrampolineAddrs)](Expected<SymbolMap> Result) mutable {
+ if (Result)
+ OnTrampolinesReady(std::move(*TrampolineAddrs));
+ else
+ OnTrampolinesReady(Result.takeError());
+ },
+ NoDependenciesToRegister);
+}
+
+Expected<std::unique_ptr<LazyReexportsManager>>
+createJITLinkLazyReexportsManager(ObjectLinkingLayer &ObjLinkingLayer,
+ RedirectableSymbolManager &RSMgr,
+ JITDylib &PlatformJD) {
+ auto JLT = JITLinkReentryTrampolines::Create(ObjLinkingLayer);
+ if (!JLT)
+ return JLT.takeError();
+
+ return LazyReexportsManager::Create(
+ [JLT = std::move(*JLT)](ResourceTrackerSP RT, size_t NumTrampolines,
+ LazyReexportsManager::OnTrampolinesReadyFn
+ OnTrampolinesReady) mutable {
+ JLT->emit(std::move(RT), NumTrampolines, std::move(OnTrampolinesReady));
+ },
+ RSMgr, PlatformJD);
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
index 1eff9a1bdf08b0..543337eb60f5d5 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyObjectLinkingLayer.cpp
@@ -71,10 +71,9 @@ class LazyObjectLinkingLayer::RenamerPlugin
};
LazyObjectLinkingLayer::LazyObjectLinkingLayer(ObjectLinkingLayer &BaseLayer,
- LazyCallThroughManager &LCTMgr,
- RedirectableSymbolManager &RSMgr)
+ LazyReexportsManager &LRMgr)
: ObjectLayer(BaseLayer.getExecutionSession()), BaseLayer(BaseLayer),
- LCTMgr(LCTMgr), RSMgr(RSMgr) {
+ LRMgr(LRMgr) {
BaseLayer.addPlugin(std::make_unique<RenamerPlugin>());
}
@@ -101,8 +100,7 @@ Error LazyObjectLinkingLayer::add(ResourceTrackerSP RT,
return Err;
auto &JD = RT->getJITDylib();
- return JD.define(lazyReexports(LCTMgr, RSMgr, JD, std::move(LazySymbols)),
- std::move(RT));
+ return JD.define(lazyReexports(LRMgr, std::move(LazySymbols)), std::move(RT));
}
void LazyObjectLinkingLayer::emit(
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
index 15c5f79fdbd3c6..7a7e5d13ce03fd 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -8,7 +8,9 @@
#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
+#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
+#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"
#include "llvm/TargetParser/Triple.h"
#define DEBUG_TYPE "orc"
@@ -229,5 +231,167 @@ LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
}
+class LazyReexportsManager::MU : public MaterializationUnit {
+public:
+ MU(LazyReexportsManager &LRMgr, SymbolAliasMap Reexports)
+ : MaterializationUnit(getInterface(Reexports)), LRMgr(LRMgr),
+ Reexports(std::move(Reexports)) {}
+
+private:
+ Interface getInterface(const SymbolAliasMap &Reexports) {
+ SymbolFlagsMap SF;
+ for (auto &[Alias, AI] : Reexports)
+ SF[Alias] = AI.AliasFlags;
+ return {std::move(SF), nullptr};
+ }
+
+ StringRef getName() const override { return "LazyReexportsManager::MU"; }
+
+ void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
+ LRMgr.emitReentryTrampolines(std::move(R), std::move(Reexports));
+ }
+
+ void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
+ Reexports.erase(Name);
+ }
+
+ LazyReexportsManager &LRMgr;
+ SymbolAliasMap Reexports;
+};
+
+class LazyReexportsManager::Plugin : public ObjectLinkingLayer::Plugin {
+public:
+ void modifyPassConfig(MaterializationResponsibility &MR,
+ jitlink::LinkGraph &G,
+ jitlink::PassConfiguration &Config) override {}
+
+ Error notifyFailed(MaterializationResponsibility &MR) override {
+ return Error::success();
+ }
+
+ Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
+ return Error::success();
+ }
+
+ void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
+ ResourceKey SrcKey) override {}
+
+private:
+ std::mutex M;
+};
+
+Expected<std::unique_ptr<LazyReexportsManager>>
+LazyReexportsManager::Create(EmitTrampolinesFn EmitTrampolines,
+ RedirectableSymbolManager &RSMgr,
+ JITDylib &PlatformJD) {
+ Error Err = Error::success();
+ std::unique_ptr<LazyReexportsManager> LRM(new LazyReexportsManager(
+ std::move(EmitTrampolines), RSMgr, PlatformJD, Err));
+ if (Err)
+ return std::move(Err);
+ return std::move(LRM);
+}
+
+LazyReexportsManager::LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
+ RedirectableSymbolManager &RSMgr,
+ JITDylib &PlatformJD, Error &Err)
+ : EmitTrampolines(std::move(EmitTrampolines)), RSMgr(RSMgr) {
+
+ using namespace shared;
+
+ ErrorAsOutParameter _(&Err);
+
+ auto &ES = PlatformJD.getExecutionSession();
+ ExecutionSession::JITDispatchHandlerAssociationMap WFs;
+
+ WFs[ES.intern("__orc_rt_resolve_tag")] =
+ ES.wrapAsyncWithSPS<SPSExpected<SPSExecutorSymbolDef>(SPSExecutorAddr)>(
+ this, &LazyReexportsManager::resolve);
+
+ Err = ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
+}
+
+std::unique_ptr<MaterializationUnit>
+LazyReexportsManager::createLazyReexports(SymbolAliasMap Reexports) {
+ return std::make_unique<MU>(*this, std::move(Reexports));
+}
+
+void LazyReexportsManager::emitReentryTrampolines(
+ std::unique_ptr<MaterializationResponsibility> MR,
+ SymbolAliasMap Reexports) {
+ size_t NumTrampolines = Reexports.size();
+ auto RT = MR->getResourceTracker();
+ EmitTrampolines(
+ std::move(RT), NumTrampolines,
+ [this, MR = std::move(MR), Reexports = std::move(Reexports)](
+ Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) mutable {
+ emitRedirectableSymbols(std::move(MR), std::move(Reexports),
+ std::move(ReentryPoints));
+ });
+}
+
+void LazyReexportsManager::emitRedirectableSymbols(
+ std::unique_ptr<MaterializationResponsibility> MR, SymbolAliasMap Reexports,
+ Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) {
+
+ if (!ReentryPoints) {
+ MR->getExecutionSession().reportError(ReentryPoints.takeError());
+ MR->failMaterialization();
+ return;
+ }
+
+ assert(Reexports.size() == ReentryPoints->size() &&
+ "Number of reentry points doesn't match number of reexports");
+
+ // Bind entry points to names.
+ SymbolMap Redirs;
+ {
+ std::lock_guard<std::mutex> Lock(M);
+ size_t I = 0;
+ for (auto &[Name, AI] : Reexports) {
+ const auto &ReentryPoint = (*ReentryPoints)[I++];
+ Redirs[Name] = ReentryPoint;
+ CallThroughs[ReentryPoint.getAddress()] = {Name, AI.Aliasee,
+ &MR->getTargetJITDylib()};
+ }
+ }
+
+ RSMgr.emitRedirectableSymbols(std::move(MR), std::move(Redirs));
+}
+
+void LazyReexportsManager::resolve(ResolveSendResultFn SendResult,
+ ExecutorAddr ReentryStubAddr) {
+
+ CallThroughInfo LandingInfo;
+
+ {
+ std::lock_guard<std::mutex> Lock(M);
+
+ auto I = CallThroughs.find(ReentryStubAddr);
+ if (I == CallThroughs.end())
+ return SendResult(make_error<StringError>(
+ "Reentry address " + formatv("{0:x}", ReentryStubAddr) +
+ " not registered",
+ inconvertibleErrorCode()));
+ LandingInfo = I->second;
+ }
+
+ SymbolInstance LandingSym(LandingInfo.JD, std::move(LandingInfo.BodyName));
+ LandingSym.lookupAsync([this, JD = std::move(LandingInfo.JD),
+ ReentryName = std::move(LandingInfo.Name),
+ SendResult = std::move(SendResult)](
+ Expected<ExecutorSymbolDef> Result) mutable {
+ if (Result) {
+ // FIXME: Make RedirectionManager operations async, then use the async
+ // APIs here.
+ if (auto Err = RSMgr.redirect(*JD, ReentryName, *Result))
+ SendResult(std::move(Err));
+ else
+ SendResult(std::move(Result));
+ } else
+ SendResult(std::move(Result));
+ });
+}
+
} // End namespace orc.
} // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index f3fba51f2e7433..0e8349711e6fed 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -409,6 +409,7 @@ SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
SymbolAliasMap Aliases;
addAliases(ES, Aliases, requiredCXXAliases());
addAliases(ES, Aliases, standardRuntimeUtilityAliases());
+ addAliases(ES, Aliases, standardLazyCompilationAliases());
return Aliases;
}
@@ -436,6 +437,17 @@ MachOPlatform::standardRuntimeUtilityAliases() {
StandardRuntimeUtilityAliases);
}
+ArrayRef<std::pair<const char *, const char *>>
+MachOPlatform::standardLazyCompilationAliases() {
+ static const std::pair<const char *, const char *>
+ StandardLazyCompilationAliases[] = {
+ {"__orc_rt_reenter", "__orc_rt_sysv_reenter"},
+ {"__orc_rt_resolve_tag", "___orc_rt_resolve_tag"}};
+
+ return ArrayRef<std::pair<const char *, const char *>>(
+ StandardLazyCompilationAliases);
+}
+
bool MachOPlatform::supportedTarget(const Triple &TT) {
switch (TT.getArch()) {
case Triple::aarch64:
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 77c07cf5cdb45c..3c58b8934462a2 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -30,6 +30,7 @@
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
#include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
+#include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"
#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
#include "llvm/ExecutionEngine/Orc/LoadLinkableFile.h"
#include "llvm/ExecutionEngine/Orc/MachO.h"
@@ -949,41 +950,18 @@ class PhonyExternalsGenerator : public DefinitionGenerator {
}
};
-static void handleLazyCallFailure() {
- dbgs() << "ERROR: failure to materialize lazy call-through target.\n";
- exit(1);
-}
-
-static void *reenter(void *Ctx, void *TrampolineAddr) {
- std::promise<void *> LandingAddressP;
- auto LandingAddressF = LandingAddressP.get_future();
-
- auto *EPCIU = static_cast<EPCIndirectionUtils *>(Ctx);
- EPCIU->getLazyCallThroughManager().resolveTrampolineLandingAddress(
- ExecutorAddr::fromPtr(TrampolineAddr), [&](ExecutorAddr LandingAddress) {
- LandingAddressP.set_value(LandingAddress.toPtr<void *>());
- });
- return LandingAddressF.get();
-}
-
Expected<std::unique_ptr<Session::LazyLinkingSupport>>
-createLazyLinkingSupport(ObjectLinkingLayer &OLL) {
- auto EPCIU = EPCIndirectionUtils::Create(OLL.getExecutionSession());
- if (!EPCIU)
- return EPCIU.takeError();
- if (auto Err = (*EPCIU)
- ->writeResolverBlock(ExecutorAddr::fromPtr(&reenter),
- ExecutorAddr::fromPtr(EPCIU->get()))
- .takeError())
- return Err;
- (*EPCIU)->createLazyCallThroughManager(
- OLL.getExecutionSession(), ExecutorAddr::fromPtr(handleLazyCallFailure));
+createLazyLinkingSupport(ObjectLinkingLayer &OLL, JITDylib &PlatformJD) {
auto RSMgr = JITLinkRedirectableSymbolManager::Create(OLL);
if (!RSMgr)
return RSMgr.takeError();
- return std::make_unique<Session::LazyLinkingSupport>(std::move(*EPCIU),
- std::move(*RSMgr), OLL);
+ auto LRMgr = createJITLinkLazyReexportsManager(OLL, **RSMgr, PlatformJD);
+ if (!LRMgr)
+ return LRMgr.takeError();
+
+ return std::make_unique<Session::LazyLinkingSupport>(std::move(*RSMgr),
+ std::move(*LRMgr), OLL);
}
Expected<std::unique_ptr<Session>> Session::Create(Triple TT,
@@ -1020,7 +998,8 @@ Expected<std::unique_ptr<Session>> Session::Create(Triple TT,
S->Features = std::move(Features);
if (lazyLinkingRequested()) {
- if (auto LazyLinking = createLazyLinkingSupport(S->ObjLayer))
+ if (auto LazyLinking =
+ createLazyLinkingSupport(S->ObjLayer, *S->PlatformJD))
S->LazyLinking = std::move(*LazyLinking);
else
return LazyLinking.takeError();
@@ -1642,10 +1621,17 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) {
OutOfProcessExecutor = OOPExecutorPath.str().str();
}
- if (lazyLinkingRequested() && !TestHarnesses.empty())
- return make_error<StringError>(
- "Lazy linking cannot be used with -harness mode",
- inconvertibleErrorCode());
+ // If lazy linking is requested then check compatibility with other options.
+ if (lazyLinkingRequested()) {
+ if (OrcRuntime.empty())
+ return make_error<StringError>("Lazy linking requries the ORC runtime",
+ inconvertibleErrorCode());
+
+ if (!TestHarnesses.empty())
+ return make_error<StringError>(
+ "Lazy linking cannot be used with -harness mode",
+ inconvertibleErrorCode());
+ }
return Error::success();
}
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h
index bdf91ea7e3f18e..bfad5211c21766 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.h
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h
@@ -15,9 +15,9 @@
#include "llvm/ADT/StringSet.h"
#include "llvm/ExecutionEngine/Orc/Core.h"
-#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
#include "llvm/ExecutionEngine/Orc/LazyObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/LazyReexports.h"
#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/RedirectionManager.h"
#include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h"
@@ -33,20 +33,14 @@ namespace llvm {
struct Session {
struct LazyLinkingSupport {
- LazyLinkingSupport(std::unique_ptr<orc::EPCIndirectionUtils> EPCIU,
- std::unique_ptr<orc::RedirectableSymbolManager> RSMgr,
+ LazyLinkingSupport(std::unique_ptr<orc::RedirectableSymbolManager> RSMgr,
+ std::unique_ptr<orc::LazyReexportsManager> LRMgr,
orc::ObjectLinkingLayer &ObjLinkingLayer)
- : EPCIU(std::move(EPCIU)), RSMgr(std::move(RSMgr)),
- LazyObjLinkingLayer(ObjLinkingLayer,
- this->EPCIU->getLazyCallThroughManager(),
- *this->RSMgr) {}
- ~LazyLinkingSupport() {
- if (auto Err = EPCIU->cleanup())
- LazyObjLinkingLayer.getExecutionSession().reportError(std::move(Err));
- }
-
- std::unique_ptr<orc::EPCIndirectionUtils> EPCIU;
+ : RSMgr(std::move(RSMgr)), LRMgr(std::move(LRMgr)),
+ LazyObjLinkingLayer(ObjLinkingLayer, *this->LRMgr) {}
+
std::unique_ptr<orc::RedirectableSymbolManager> RSMgr;
+ std::unique_ptr<orc::LazyReexportsManager> LRMgr;
orc::LazyObjectLinkingLayer LazyObjLinkingLayer;
};
More information about the llvm-commits
mailing list