[llvm] 5acd471 - [ORC] Add a shared-memory based orc::MemoryMapper.
Lang Hames via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 13 15:24:35 PDT 2022
Author: Anubhab Ghosh
Date: 2022-07-13T15:24:28-07:00
New Revision: 5acd471698849d9e322a29e6ca08791e8d447b7b
URL: https://github.com/llvm/llvm-project/commit/5acd471698849d9e322a29e6ca08791e8d447b7b
DIFF: https://github.com/llvm/llvm-project/commit/5acd471698849d9e322a29e6ca08791e8d447b7b.diff
LOG: [ORC] Add a shared-memory based orc::MemoryMapper.
This is an implementation of orc::MemoryMapper that maps shared memory
pages in both executor and controller process and writes directly to
them avoiding transferring content over EPC. All allocations are properly
deinitialized automatically on the executor side at shutdown by the
ExecutorSharedMemoryMapperService.
Reviewed By: lhames
Differential Revision: https://reviews.llvm.org/D128544
Added:
llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h
llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp
llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
Modified:
llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
index d023bfbdb5b6f..ba27bceb89884 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
@@ -109,6 +109,47 @@ class InProcessMemoryMapper final : public MemoryMapper {
AllocationMap Allocations;
};
+class SharedMemoryMapper final : public MemoryMapper {
+public:
+ struct SymbolAddrs {
+ ExecutorAddr Instance;
+ ExecutorAddr Reserve;
+ ExecutorAddr Initialize;
+ ExecutorAddr Deinitialize;
+ ExecutorAddr Release;
+ };
+
+ SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs)
+ : EPC(EPC), SAs(SAs) {}
+
+ void reserve(size_t NumBytes, OnReservedFunction OnReserved) override;
+
+ char *prepare(ExecutorAddr Addr, size_t ContentSize) override;
+
+ void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override;
+
+ void deinitialize(ArrayRef<ExecutorAddr> Allocations,
+ OnDeinitializedFunction OnDeInitialized) override;
+
+ void release(ArrayRef<ExecutorAddr> Reservations,
+ OnReleasedFunction OnRelease) override;
+
+ ~SharedMemoryMapper() override;
+
+private:
+ struct Reservation {
+ void *LocalAddr;
+ size_t Size;
+ };
+
+ ExecutorProcessControl &EPC;
+ SymbolAddrs SAs;
+
+ std::mutex Mutex;
+
+ std::map<ExecutorAddr, Reservation> Reservations;
+};
+
} // namespace orc
} // end namespace llvm
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index 96166ac20b2e3..74fb803071202 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -31,6 +31,12 @@ extern const char *SimpleExecutorMemoryManagerReserveWrapperName;
extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName;
extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName;
+extern const char *ExecutorSharedMemoryMapperServiceInstanceName;
+extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName;
+extern const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName;
+extern const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName;
+extern const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName;
+
extern const char *MemoryWriteUInt8sWrapperName;
extern const char *MemoryWriteUInt16sWrapperName;
extern const char *MemoryWriteUInt32sWrapperName;
@@ -58,6 +64,21 @@ using SPSSimpleExecutorMemoryManagerFinalizeSignature =
using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError(
shared::SPSExecutorAddr, shared::SPSSequence<shared::SPSExecutorAddr>);
+// ExecutorSharedMemoryMapperService
+using SPSExecutorSharedMemoryMapperServiceReserveSignature =
+ shared::SPSExpected<
+ shared::SPSTuple<shared::SPSExecutorAddr, shared::SPSString>>(
+ shared::SPSExecutorAddr, uint64_t);
+using SPSExecutorSharedMemoryMapperServiceInitializeSignature =
+ shared::SPSExpected<shared::SPSExecutorAddr>(shared::SPSExecutorAddr,
+ shared::SPSExecutorAddr,
+ shared::SPSFinalizeRequest);
+using SPSExecutorSharedMemoryMapperServiceDeinitializeSignature =
+ shared::SPSError(shared::SPSExecutorAddr,
+ shared::SPSSequence<shared::SPSExecutorAddr>);
+using SPSExecutorSharedMemoryMapperServiceReleaseSignature = shared::SPSError(
+ shared::SPSExecutorAddr, shared::SPSSequence<shared::SPSExecutorAddr>);
+
using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr,
shared::SPSSequence<shared::SPSString>);
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h
new file mode 100644
index 0000000000000..15610ce598539
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h
@@ -0,0 +1,78 @@
+//===----------- ExecutorSharedMemoryMapperService.h ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE
+#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
+
+#include <atomic>
+#include <mutex>
+
+#if defined(_WIN32)
+#include <windows.h>
+#endif
+
+namespace llvm {
+namespace orc {
+namespace rt_bootstrap {
+
+class ExecutorSharedMemoryMapperService final
+ : public ExecutorBootstrapService {
+public:
+ ~ExecutorSharedMemoryMapperService(){};
+
+ Expected<std::pair<ExecutorAddr, std::string>> reserve(uint64_t Size);
+ Expected<ExecutorAddr> initialize(ExecutorAddr Reservation,
+ tpctypes::FinalizeRequest &FR);
+
+ Error deinitialize(const std::vector<ExecutorAddr> &Bases);
+ Error release(const std::vector<ExecutorAddr> &Bases);
+
+ Error shutdown() override;
+ void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override;
+
+private:
+ struct Allocation {
+ std::vector<shared::WrapperFunctionCall> DeinitializationActions;
+ };
+ using AllocationMap = DenseMap<ExecutorAddr, Allocation>;
+
+ struct Reservation {
+ size_t Size;
+ std::vector<ExecutorAddr> Allocations;
+#if defined(_WIN32)
+ HANDLE SharedMemoryFile;
+#endif
+ };
+ using ReservationMap = DenseMap<void *, Reservation>;
+
+ static llvm::orc::shared::CWrapperFunctionResult
+ reserveWrapper(const char *ArgData, size_t ArgSize);
+
+ static llvm::orc::shared::CWrapperFunctionResult
+ initializeWrapper(const char *ArgData, size_t ArgSize);
+
+ static llvm::orc::shared::CWrapperFunctionResult
+ deinitializeWrapper(const char *ArgData, size_t ArgSize);
+
+ static llvm::orc::shared::CWrapperFunctionResult
+ releaseWrapper(const char *ArgData, size_t ArgSize);
+
+ std::atomic<int> SharedMemoryCount{0};
+ std::mutex Mutex;
+ ReservationMap Reservations;
+ AllocationMap Allocations;
+};
+
+} // namespace rt_bootstrap
+} // namespace orc
+} // namespace llvm
+#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE
diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
index 8b3fbd7117e2a..2960ce2e85510 100644
--- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
@@ -8,6 +8,17 @@
#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/Support/WindowsError.h"
+
+#if defined(LLVM_ON_UNIX)
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#elif defined(_WIN32)
+#include <windows.h>
+#endif
+
namespace llvm {
namespace orc {
@@ -147,6 +158,225 @@ InProcessMemoryMapper::~InProcessMemoryMapper() {
cantFail(F.get());
}
+// SharedMemoryMapper
+
+void SharedMemoryMapper::reserve(size_t NumBytes,
+ OnReservedFunction OnReserved) {
+#if defined(LLVM_ON_UNIX) || defined(_WIN32)
+
+ EPC.callSPSWrapperAsync<
+ rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
+ SAs.Reserve,
+ [this, NumBytes, OnReserved = std::move(OnReserved)](
+ Error SerializationErr,
+ Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
+ if (SerializationErr) {
+ cantFail(Result.takeError());
+ return OnReserved(std::move(SerializationErr));
+ }
+
+ if (!Result)
+ return OnReserved(Result.takeError());
+
+ ExecutorAddr RemoteAddr;
+ std::string SharedMemoryName;
+ std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
+
+ void *LocalAddr = nullptr;
+
+#if defined(LLVM_ON_UNIX)
+
+ int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
+ if (SharedMemoryFile < 0) {
+ return OnReserved(errorCodeToError(
+ std::error_code(errno, std::generic_category())));
+ }
+
+ // this prevents other processes from accessing it by name
+ shm_unlink(SharedMemoryName.c_str());
+
+ LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
+ SharedMemoryFile, 0);
+ if (LocalAddr == MAP_FAILED) {
+ return OnReserved(errorCodeToError(
+ std::error_code(errno, std::generic_category())));
+ }
+
+ close(SharedMemoryFile);
+
+#elif defined(_WIN32)
+
+ std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
+ SharedMemoryName.end());
+ HANDLE SharedMemoryFile = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE,
+ WideSharedMemoryName.c_str());
+ if (!SharedMemoryFile)
+ return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
+
+ LocalAddr =
+ MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
+ if (!LocalAddr) {
+ CloseHandle(SharedMemoryFile);
+ return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
+ }
+
+ CloseHandle(SharedMemoryFile);
+
+#endif
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
+ }
+
+ OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
+ },
+ SAs.Instance, static_cast<uint64_t>(NumBytes));
+
+#else
+ OnReserved(make_error<StringError>(
+ "SharedMemoryMapper is not supported on this platform yet",
+ inconvertibleErrorCode()));
+#endif
+}
+
+char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
+ auto R = Reservations.upper_bound(Addr);
+ assert(R != Reservations.begin() && "Attempt to prepare unknown range");
+ R--;
+
+ ExecutorAddrDiff Offset = Addr - R->first;
+
+ return static_cast<char *>(R->second.LocalAddr) + Offset;
+}
+
+void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
+ OnInitializedFunction OnInitialized) {
+ auto Reservation = Reservations.find(AI.MappingBase);
+ assert(Reservation != Reservations.end() &&
+ "Attempt to initialize unreserved range");
+
+ tpctypes::FinalizeRequest FR;
+
+ AI.Actions.swap(FR.Actions);
+
+ FR.Segments.reserve(AI.Segments.size());
+
+ for (auto Segment : AI.Segments) {
+ char *Base =
+ static_cast<char *>(Reservation->second.LocalAddr) + Segment.Offset;
+ std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
+
+ tpctypes::SegFinalizeRequest SegReq;
+ SegReq.Prot = tpctypes::toWireProtectionFlags(
+ static_cast<sys::Memory::ProtectionFlags>(Segment.Prot));
+ SegReq.Addr = AI.MappingBase + Segment.Offset;
+ SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
+
+ FR.Segments.push_back(SegReq);
+ }
+
+ EPC.callSPSWrapperAsync<
+ rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
+ SAs.Initialize,
+ [OnInitialized = std::move(OnInitialized)](
+ Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
+ if (SerializationErr) {
+ cantFail(Result.takeError());
+ return OnInitialized(std::move(SerializationErr));
+ }
+
+ OnInitialized(std::move(Result));
+ },
+ SAs.Instance, AI.MappingBase, std::move(FR));
+}
+
+void SharedMemoryMapper::deinitialize(
+ ArrayRef<ExecutorAddr> Allocations,
+ MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
+ EPC.callSPSWrapperAsync<
+ rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
+ SAs.Deinitialize,
+ [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
+ Error Result) mutable {
+ if (SerializationErr) {
+ cantFail(std::move(Result));
+ return OnDeinitialized(std::move(SerializationErr));
+ }
+
+ OnDeinitialized(std::move(Result));
+ },
+ SAs.Instance, Allocations);
+}
+
+void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
+ OnReleasedFunction OnReleased) {
+#if defined(LLVM_ON_UNIX) || defined(_WIN32)
+ Error Err = Error::success();
+
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ for (auto Base : Bases) {
+
+#if defined(LLVM_ON_UNIX)
+
+ if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
+ Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
+ errno, std::generic_category())));
+
+#elif defined(_WIN32)
+
+ if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
+ joinErrors(std::move(Err),
+ errorCodeToError(mapWindowsError(GetLastError())));
+
+#endif
+
+ Reservations.erase(Base);
+ }
+ }
+
+ EPC.callSPSWrapperAsync<
+ rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
+ SAs.Release,
+ [OnReleased = std::move(OnReleased),
+ Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
+ if (SerializationErr) {
+ cantFail(std::move(Result));
+ return OnReleased(
+ joinErrors(std::move(Err), std::move(SerializationErr)));
+ }
+
+ return OnReleased(joinErrors(std::move(Err), std::move(Result)));
+ },
+ SAs.Instance, Bases);
+#else
+ OnReleased(make_error<StringError>(
+ "SharedMemoryMapper is not supported on this platform yet",
+ inconvertibleErrorCode()));
+#endif
+}
+
+SharedMemoryMapper::~SharedMemoryMapper() {
+ std::vector<ExecutorAddr> ReservationAddrs;
+ if (!Reservations.empty()) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ {
+ ReservationAddrs.reserve(Reservations.size());
+ for (const auto &R : Reservations) {
+ ReservationAddrs.push_back(R.first);
+ }
+ }
+ }
+
+ std::promise<MSVCPError> P;
+ auto F = P.get_future();
+ release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
+ // FIXME: Release can actually fail. The error should be propagated.
+ // Meanwhile, a better option is to explicitly call release().
+ cantFail(F.get());
+}
+
} // namespace orc
} // namespace llvm
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
index 5eae33121eb9c..dfdd846c46a75 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
@@ -18,6 +18,7 @@ const char *SimpleExecutorDylibManagerOpenWrapperName =
"__llvm_orc_SimpleExecutorDylibManager_open_wrapper";
const char *SimpleExecutorDylibManagerLookupWrapperName =
"__llvm_orc_SimpleExecutorDylibManager_lookup_wrapper";
+
const char *SimpleExecutorMemoryManagerInstanceName =
"__llvm_orc_SimpleExecutorMemoryManager_Instance";
const char *SimpleExecutorMemoryManagerReserveWrapperName =
@@ -26,6 +27,18 @@ const char *SimpleExecutorMemoryManagerFinalizeWrapperName =
"__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper";
const char *SimpleExecutorMemoryManagerDeallocateWrapperName =
"__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper";
+
+const char *ExecutorSharedMemoryMapperServiceInstanceName =
+ "__llvm_orc_ExecutorSharedMemoryMapperService_Instance";
+const char *ExecutorSharedMemoryMapperServiceReserveWrapperName =
+ "__llvm_orc_ExecutorSharedMemoryMapperService_Reserve";
+const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName =
+ "__llvm_orc_ExecutorSharedMemoryMapperService_Initialize";
+const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName =
+ "__llvm_orc_ExecutorSharedMemoryMapperService_Deinitialize";
+const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName =
+ "__llvm_orc_ExecutorSharedMemoryMapperService_Release";
+
const char *MemoryWriteUInt8sWrapperName =
"__llvm_orc_bootstrap_mem_write_uint8s_wrapper";
const char *MemoryWriteUInt16sWrapperName =
@@ -36,10 +49,12 @@ const char *MemoryWriteUInt64sWrapperName =
"__llvm_orc_bootstrap_mem_write_uint64s_wrapper";
const char *MemoryWriteBuffersWrapperName =
"__llvm_orc_bootstrap_mem_write_buffers_wrapper";
+
const char *RegisterEHFrameSectionWrapperName =
"__llvm_orc_bootstrap_register_ehframe_section_wrapper";
const char *DeregisterEHFrameSectionWrapperName =
"__llvm_orc_bootstrap_deregister_ehframe_section_wrapper";
+
const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper";
} // end namespace rt
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
index c4edece96cb04..415876abd65f5 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt
@@ -1,4 +1,5 @@
add_llvm_component_library(LLVMOrcTargetProcess
+ ExecutorSharedMemoryMapperService.cpp
JITLoaderGDB.cpp
OrcRTBootstrap.cpp
RegisterEHFrames.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp
new file mode 100644
index 0000000000000..a1dae2ad72be2
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp
@@ -0,0 +1,342 @@
+//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
+
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/WindowsError.h"
+
+#include <sstream>
+
+#if defined(LLVM_ON_UNIX)
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
+
+#if defined(_WIN32)
+static DWORD getWindowsProtectionFlags(unsigned Flags) {
+ switch (Flags & llvm::sys::Memory::MF_RWE_MASK) {
+ case llvm::sys::Memory::MF_READ:
+ return PAGE_READONLY;
+ case llvm::sys::Memory::MF_WRITE:
+ // Note: PAGE_WRITE is not supported by VirtualProtect
+ return PAGE_READWRITE;
+ case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE:
+ return PAGE_READWRITE;
+ case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_EXEC:
+ return PAGE_EXECUTE_READ;
+ case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE |
+ llvm::sys::Memory::MF_EXEC:
+ return PAGE_EXECUTE_READWRITE;
+ case llvm::sys::Memory::MF_EXEC:
+ return PAGE_EXECUTE;
+ default:
+ llvm_unreachable("Illegal memory protection flag specified!");
+ }
+ // Provide a default return value as required by some compilers.
+ return PAGE_NOACCESS;
+}
+#endif
+
+namespace llvm {
+namespace orc {
+namespace rt_bootstrap {
+
+Expected<std::pair<ExecutorAddr, std::string>>
+ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
+#if defined(LLVM_ON_UNIX) || defined(_WIN32)
+
+#if defined(LLVM_ON_UNIX)
+
+ std::string SharedMemoryName;
+ {
+ std::stringstream SharedMemoryNameStream;
+ SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
+ << (++SharedMemoryCount);
+ SharedMemoryName = SharedMemoryNameStream.str();
+ }
+
+ int SharedMemoryFile =
+ shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);
+ if (SharedMemoryFile < 0)
+ return errorCodeToError(std::error_code(errno, std::generic_category()));
+
+ // by default size is 0
+ if (ftruncate(SharedMemoryFile, Size) < 0)
+ return errorCodeToError(std::error_code(errno, std::generic_category()));
+
+ void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);
+ if (Addr == MAP_FAILED)
+ return errorCodeToError(std::error_code(errno, std::generic_category()));
+
+ close(SharedMemoryFile);
+
+#elif defined(_WIN32)
+
+ std::string SharedMemoryName;
+ {
+ std::stringstream SharedMemoryNameStream;
+ SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
+ << (++SharedMemoryCount);
+ SharedMemoryName = SharedMemoryNameStream.str();
+ }
+
+ std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
+ SharedMemoryName.end());
+ HANDLE SharedMemoryFile = CreateFileMapping(
+ INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
+ Size & 0xffffffff, WideSharedMemoryName.c_str());
+ if (!SharedMemoryFile)
+ return errorCodeToError(mapWindowsError(GetLastError()));
+
+ void *Addr = MapViewOfFile(SharedMemoryFile,
+ FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
+ if (!Addr) {
+ CloseHandle(SharedMemoryFile);
+ return errorCodeToError(mapWindowsError(GetLastError()));
+ }
+
+#endif
+
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Reservations[Addr].Size = Size;
+#if defined(_WIN32)
+ Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
+#endif
+ }
+
+ return std::make_pair(ExecutorAddr::fromPtr(Addr),
+ std::move(SharedMemoryName));
+#else
+ return make_error<StringError>(
+ "SharedMemoryMapper is not supported on this platform yet",
+ inconvertibleErrorCode());
+#endif
+}
+
+Expected<ExecutorAddr>
+ExecutorSharedMemoryMapperService::initialize(ExecutorAddr Reservation,
+ tpctypes::FinalizeRequest &FR) {
+#if defined(LLVM_ON_UNIX) || defined(_WIN32)
+
+ ExecutorAddr MinAddr(~0ULL);
+
+ // Contents are already in place
+ for (auto &Segment : FR.Segments) {
+ if (Segment.Addr < MinAddr)
+ MinAddr = Segment.Addr;
+
+#if defined(LLVM_ON_UNIX)
+
+ int NativeProt = 0;
+ if (Segment.Prot & tpctypes::WPF_Read)
+ NativeProt |= PROT_READ;
+ if (Segment.Prot & tpctypes::WPF_Write)
+ NativeProt |= PROT_WRITE;
+ if (Segment.Prot & tpctypes::WPF_Exec)
+ NativeProt |= PROT_EXEC;
+
+ if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))
+ return errorCodeToError(std::error_code(errno, std::generic_category()));
+
+#elif defined(_WIN32)
+
+ DWORD NativeProt =
+ getWindowsProtectionFlags(fromWireProtectionFlags(Segment.Prot));
+
+ if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
+ &NativeProt))
+ return errorCodeToError(mapWindowsError(GetLastError()));
+
+#endif
+
+ if (Segment.Prot & tpctypes::WPF_Exec)
+ sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),
+ Segment.Size);
+ }
+
+ // Run finalization actions and get deinitlization action list.
+ auto DeinitializeActions = shared::runFinalizeActions(FR.Actions);
+ if (!DeinitializeActions) {
+ return DeinitializeActions.takeError();
+ }
+
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Allocations[MinAddr].DeinitializationActions =
+ std::move(*DeinitializeActions);
+ Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);
+ }
+
+ return MinAddr;
+
+#else
+ return make_error<StringError>(
+ "SharedMemoryMapper is not supported on this platform yet",
+ inconvertibleErrorCode());
+#endif
+}
+
+Error ExecutorSharedMemoryMapperService::deinitialize(
+ const std::vector<ExecutorAddr> &Bases) {
+ Error AllErr = Error::success();
+
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ for (auto Base : Bases) {
+ if (Error Err = shared::runDeallocActions(
+ Allocations[Base].DeinitializationActions)) {
+ AllErr = joinErrors(std::move(AllErr), std::move(Err));
+ }
+
+ Allocations.erase(Base);
+ }
+ }
+
+ return AllErr;
+}
+
+Error ExecutorSharedMemoryMapperService::release(
+ const std::vector<ExecutorAddr> &Bases) {
+#if defined(LLVM_ON_UNIX) || defined(_WIN32)
+ Error Err = Error::success();
+
+ for (auto Base : Bases) {
+ std::vector<ExecutorAddr> AllocAddrs;
+ size_t Size;
+
+#if defined(_WIN32)
+ HANDLE SharedMemoryFile;
+#endif
+
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ auto &R = Reservations[Base.toPtr<void *>()];
+ Size = R.Size;
+
+#if defined(_WIN32)
+ SharedMemoryFile = R.SharedMemoryFile;
+#endif
+
+ AllocAddrs.swap(R.Allocations);
+ }
+
+ // deinitialize sub allocations
+ if (Error E = deinitialize(AllocAddrs))
+ Err = joinErrors(std::move(Err), std::move(E));
+
+#if defined(LLVM_ON_UNIX)
+
+ if (munmap(Base.toPtr<void *>(), Size) != 0)
+ Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
+ errno, std::generic_category())));
+
+#elif defined(_WIN32)
+
+ if (!UnmapViewOfFile(Base.toPtr<void *>()))
+ Err = joinErrors(std::move(Err),
+ errorCodeToError(mapWindowsError(GetLastError())));
+
+ CloseHandle(SharedMemoryFile);
+
+#endif
+
+ std::lock_guard<std::mutex> Lock(Mutex);
+ Reservations.erase(Base.toPtr<void *>());
+ }
+
+ return Err;
+#else
+ return make_error<StringError>(
+ "SharedMemoryMapper is not supported on this platform yet",
+ inconvertibleErrorCode());
+#endif
+}
+
+Error ExecutorSharedMemoryMapperService::shutdown() {
+ std::vector<ExecutorAddr> ReservationAddrs;
+ if (!Reservations.empty()) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ {
+ ReservationAddrs.reserve(Reservations.size());
+ for (const auto &R : Reservations) {
+ ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
+ }
+ }
+ }
+ return release(ReservationAddrs);
+
+ return Error::success();
+}
+
+void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
+ StringMap<ExecutorAddr> &M) {
+ M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
+ ExecutorAddr::fromPtr(this);
+ M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
+ ExecutorAddr::fromPtr(&reserveWrapper);
+ M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
+ ExecutorAddr::fromPtr(&initializeWrapper);
+ M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
+ ExecutorAddr::fromPtr(&deinitializeWrapper);
+ M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
+ ExecutorAddr::fromPtr(&releaseWrapper);
+}
+
+llvm::orc::shared::CWrapperFunctionResult
+ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return shared::WrapperFunction<
+ rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
+ handle(ArgData, ArgSize,
+ shared::makeMethodWrapperHandler(
+ &ExecutorSharedMemoryMapperService::reserve))
+ .release();
+}
+
+llvm::orc::shared::CWrapperFunctionResult
+ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return shared::WrapperFunction<
+ rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
+ handle(ArgData, ArgSize,
+ shared::makeMethodWrapperHandler(
+ &ExecutorSharedMemoryMapperService::initialize))
+ .release();
+}
+
+llvm::orc::shared::CWrapperFunctionResult
+ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return shared::WrapperFunction<
+ rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
+ handle(ArgData, ArgSize,
+ shared::makeMethodWrapperHandler(
+ &ExecutorSharedMemoryMapperService::deinitialize))
+ .release();
+}
+
+llvm::orc::shared::CWrapperFunctionResult
+ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return shared::WrapperFunction<
+ rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
+ handle(ArgData, ArgSize,
+ shared::makeMethodWrapperHandler(
+ &ExecutorSharedMemoryMapperService::release))
+ .release();
+}
+
+} // namespace rt_bootstrap
+} // end namespace orc
+} // end namespace llvm
diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
index dd4028e5302ca..376ce82474854 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_unittest(OrcJITTests
OrcTestCommon.cpp
ResourceTrackerTest.cpp
RTDyldObjectLinkingLayerTest.cpp
+ SharedMemoryMapperTest.cpp
SimpleExecutorMemoryManagerTest.cpp
SimplePackedSerializationTest.cpp
SymbolStringPoolTest.cpp
diff --git a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
new file mode 100644
index 0000000000000..d7e40eec1df4d
--- /dev/null
+++ b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
@@ -0,0 +1,129 @@
+//===- SharedMemoryMapperTest.cpp -- Tests for SharedMemoryMapper ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "OrcTestCommon.h"
+
+#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
+#include "llvm/Testing/Support/Error.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+using namespace llvm::orc::shared;
+using namespace llvm::orc::rt_bootstrap;
+
+// A basic function to be used as both initializer/deinitializer
+orc::shared::CWrapperFunctionResult incrementWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
+ ArgData, ArgSize,
+ [](ExecutorAddr A) -> Error {
+ *A.toPtr<int *>() += 1;
+ return Error::success();
+ })
+ .release();
+}
+
+TEST(SharedMemoryMapperTest, MemReserveInitializeDeinitializeRelease) {
+ // These counters are used to track how many times the initializer and
+ // deinitializer functions are called
+ int InitializeCounter = 0;
+ int DeinitializeCounter = 0;
+
+ auto SelfEPC = cantFail(SelfExecutorProcessControl::Create());
+
+ ExecutorSharedMemoryMapperService MapperService;
+
+ SharedMemoryMapper::SymbolAddrs SAs;
+ {
+ StringMap<ExecutorAddr> Map;
+ MapperService.addBootstrapSymbols(Map);
+ SAs.Instance = Map[rt::ExecutorSharedMemoryMapperServiceInstanceName];
+ SAs.Reserve = Map[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName];
+ SAs.Initialize =
+ Map[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName];
+ SAs.Deinitialize =
+ Map[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName];
+ SAs.Release = Map[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName];
+ }
+
+ std::string TestString = "Hello, World!";
+
+ // barrier
+ std::promise<void> P;
+ auto F = P.get_future();
+
+ {
+ auto PageSize = cantFail(sys::Process::getPageSize());
+ size_t ReqSize = PageSize;
+
+ std::unique_ptr<MemoryMapper> Mapper =
+ std::make_unique<SharedMemoryMapper>(*SelfEPC, SAs);
+
+ Mapper->reserve(ReqSize, [&](Expected<ExecutorAddrRange> Result) {
+ EXPECT_THAT_ERROR(Result.takeError(), Succeeded());
+ auto Reservation = std::move(*Result);
+ {
+ char *Addr = Mapper->prepare(Reservation.Start, TestString.size() + 1);
+ std::strcpy(Addr, TestString.c_str());
+ }
+ MemoryMapper::AllocInfo AI;
+ {
+ MemoryMapper::AllocInfo::SegInfo SI;
+ SI.Offset = 0;
+ SI.ContentSize = TestString.size() + 1;
+ SI.ZeroFillSize = PageSize - SI.ContentSize;
+ SI.Prot = sys::Memory::MF_READ | sys::Memory::MF_WRITE;
+
+ AI.MappingBase = Reservation.Start;
+ AI.Segments.push_back(SI);
+ AI.Actions.push_back(
+ {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
+ ExecutorAddr::fromPtr(incrementWrapper),
+ ExecutorAddr::fromPtr(&InitializeCounter))),
+ cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
+ ExecutorAddr::fromPtr(incrementWrapper),
+ ExecutorAddr::fromPtr(&DeinitializeCounter)))});
+ }
+
+ EXPECT_EQ(InitializeCounter, 0);
+ EXPECT_EQ(DeinitializeCounter, 0);
+
+ Mapper->initialize(AI, [&, Reservation](Expected<ExecutorAddr> Result) {
+ EXPECT_THAT_ERROR(Result.takeError(), Succeeded());
+
+ EXPECT_EQ(TestString, std::string(static_cast<char *>(
+ Reservation.Start.toPtr<char *>())));
+
+ EXPECT_EQ(InitializeCounter, 1);
+ EXPECT_EQ(DeinitializeCounter, 0);
+
+ Mapper->deinitialize({*Result}, [&, Reservation](Error Err) {
+ EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+
+ EXPECT_EQ(InitializeCounter, 1);
+ EXPECT_EQ(DeinitializeCounter, 1);
+
+ Mapper->release({Reservation.Start}, [&](Error Err) {
+ EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+
+ P.set_value();
+ });
+ });
+ });
+ });
+
+ // This will block the test if any of the above callbacks are not executed
+ F.wait();
+ // Mapper must be destructed before calling shutdown to avoid double free
+ }
+
+ EXPECT_THAT_ERROR(MapperService.shutdown(), Succeeded());
+ cantFail(SelfEPC->disconnect());
+}
More information about the llvm-commits
mailing list