[Openmp-commits] [openmp] [Libomptarget] Remove global ctor and use reference counting (PR #80499)
Joseph Huber via Openmp-commits
openmp-commits at lists.llvm.org
Tue Feb 6 08:54:58 PST 2024
https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/80499
>From d26ab1d9798abc337509dc70e76e931682c6d359 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Fri, 2 Feb 2024 16:04:01 -0600
Subject: [PATCH] [Libomptarget] Remove global ctor and use reference counting
Summary:
Currently we rely on global constructors to initialize and shut down the
OpenMP runtime library and plugin manager. This causes some issues
because we do not have a defined lifetime that we can rely on to release
and allocate resources. This patch instead adds some simple reference
counted initialization and deinitialization function.
A future patch will use the `deinit` interface to more intelligently
handle plugin deinitilization. Right now we do nothing and rely on
`atexit` inside of the plugins to tear them down. This isn't great
because it limits our ability to control these things.
Note that I made the `__tgt_register_lib` functions do the
initialization instead of adding calls to the new runtime functions in
the linker wrapper. The reason for this is because in the past it's been
easier to not introduce a new function call, since sometimes the user's
compiler will link against an older `libomptarget`. Maybe if we change
the name with offloading in the future we can simplify this.
Depends on https://github.com/llvm/llvm-project/pull/80460
---
openmp/libomptarget/include/PluginManager.h | 6 ++++
openmp/libomptarget/include/omptarget.h | 6 ++++
openmp/libomptarget/src/OffloadRTL.cpp | 33 ++++++++++++-------
openmp/libomptarget/src/PluginManager.cpp | 2 +-
openmp/libomptarget/src/exports | 2 ++
openmp/libomptarget/src/interface.cpp | 21 +++++++++++-
.../test/offloading/runtime_init.c | 24 ++++++++++++++
7 files changed, 81 insertions(+), 13 deletions(-)
create mode 100644 openmp/libomptarget/test/offloading/runtime_init.c
diff --git a/openmp/libomptarget/include/PluginManager.h b/openmp/libomptarget/include/PluginManager.h
index ec5d98dc8cd30..5e5306ac776f0 100644
--- a/openmp/libomptarget/include/PluginManager.h
+++ b/openmp/libomptarget/include/PluginManager.h
@@ -206,6 +206,12 @@ struct PluginManager {
ProtectedObj<DeviceContainerTy> Devices;
};
+/// Initialize the plugin manager and OpenMP runtime.
+void initRuntime();
+
+/// Deinitialize the plugin and delete it.
+void deinitRuntime();
+
extern PluginManager *PM;
#endif // OMPTARGET_PLUGIN_MANAGER_H
diff --git a/openmp/libomptarget/include/omptarget.h b/openmp/libomptarget/include/omptarget.h
index 3016467b3abdf..fa31e4a480f63 100644
--- a/openmp/libomptarget/include/omptarget.h
+++ b/openmp/libomptarget/include/omptarget.h
@@ -310,6 +310,12 @@ void *llvm_omp_target_dynamic_shared_alloc();
/// add the clauses of the requires directives in a given file
void __tgt_register_requires(int64_t Flags);
+/// Initializes the runtime library.
+void __tgt_rtl_init();
+
+/// Deinitializes the runtime library.
+void __tgt_rtl_deinit();
+
/// adds a target shared library to the target execution image
void __tgt_register_lib(__tgt_bin_desc *Desc);
diff --git a/openmp/libomptarget/src/OffloadRTL.cpp b/openmp/libomptarget/src/OffloadRTL.cpp
index 86ef0d5bc91cf..b5cfd273d724b 100644
--- a/openmp/libomptarget/src/OffloadRTL.cpp
+++ b/openmp/libomptarget/src/OffloadRTL.cpp
@@ -20,25 +20,36 @@
extern void llvm::omp::target::ompt::connectLibrary();
#endif
-__attribute__((constructor(101))) void init() {
+static std::mutex PluginMtx;
+static uint32_t RefCount = 0;
+
+void initRuntime() {
+ std::scoped_lock<decltype(PluginMtx)> Lock(PluginMtx);
Profiler::get();
TIMESCOPE();
- DP("Init offload library!\n");
-
- PM = new PluginManager();
+ if (PM == nullptr) {
+ DP("Init offload library!\n");
+ PM = new PluginManager();
#ifdef OMPT_SUPPORT
- // Initialize OMPT first
- llvm::omp::target::ompt::connectLibrary();
+ // Initialize OMPT first
+ llvm::omp::target::ompt::connectLibrary();
#endif
- PM->init();
+ PM->init();
+ PM->registerDelayedLibraries();
+ }
- PM->registerDelayedLibraries();
+ RefCount++;
}
-__attribute__((destructor(101))) void deinit() {
- DP("Deinit offload library!\n");
- delete PM;
+void deinitRuntime() {
+ std::scoped_lock<decltype(PluginMtx)> Lock(PluginMtx);
+ assert(PM && "Runtime not initialized");
+
+ if (RefCount-- == 0) {
+ DP("Deinit offload library!\n");
+ delete PM;
+ }
}
diff --git a/openmp/libomptarget/src/PluginManager.cpp b/openmp/libomptarget/src/PluginManager.cpp
index 0693d4bd6c91e..4adb47ba69603 100644
--- a/openmp/libomptarget/src/PluginManager.cpp
+++ b/openmp/libomptarget/src/PluginManager.cpp
@@ -21,7 +21,7 @@
using namespace llvm;
using namespace llvm::sys;
-PluginManager *PM;
+PluginManager *PM = nullptr;
// List of all plugins that can support offloading.
static const char *RTLNames[] = {ENABLED_OFFLOAD_PLUGINS};
diff --git a/openmp/libomptarget/src/exports b/openmp/libomptarget/src/exports
index af882a2642647..d5432a9eed380 100644
--- a/openmp/libomptarget/src/exports
+++ b/openmp/libomptarget/src/exports
@@ -1,5 +1,7 @@
VERS1.0 {
global:
+ __tgt_rtl_init;
+ __tgt_rtl_deinit;
__tgt_register_requires;
__tgt_register_lib;
__tgt_unregister_lib;
diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp
index 49495ac266f1b..cbed542def0e4 100644
--- a/openmp/libomptarget/src/interface.cpp
+++ b/openmp/libomptarget/src/interface.cpp
@@ -36,9 +36,13 @@ EXTERN void __tgt_register_requires(int64_t Flags) {
PM->addRequirements(Flags);
}
+EXTERN void __tgt_rtl_init() { initRuntime(); }
+EXTERN void __tgt_rtl_deinit() { deinitRuntime(); }
+
////////////////////////////////////////////////////////////////////////////////
/// adds a target shared library to the target execution image
EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) {
+ initRuntime();
if (PM->delayRegisterLib(Desc))
return;
@@ -47,12 +51,17 @@ EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) {
////////////////////////////////////////////////////////////////////////////////
/// Initialize all available devices without registering any image
-EXTERN void __tgt_init_all_rtls() { PM->initAllPlugins(); }
+EXTERN void __tgt_init_all_rtls() {
+ assert(PM && "Runtime not initialized");
+ PM->initAllPlugins();
+}
////////////////////////////////////////////////////////////////////////////////
/// unloads a target shared library
EXTERN void __tgt_unregister_lib(__tgt_bin_desc *Desc) {
PM->unregisterLib(Desc);
+
+ deinitRuntime();
}
template <typename TargetAsyncInfoTy>
@@ -62,6 +71,7 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
map_var_info_t *ArgNames, void **ArgMappers,
TargetDataFuncPtrTy TargetDataFunction, const char *RegionTypeMsg,
const char *RegionName) {
+ assert(PM && "Runtime not initialized");
static_assert(std::is_convertible_v<TargetAsyncInfoTy, AsyncInfoTy>,
"TargetAsyncInfoTy must be convertible to AsyncInfoTy.");
@@ -236,6 +246,7 @@ template <typename TargetAsyncInfoTy>
static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
int32_t ThreadLimit, void *HostPtr,
KernelArgsTy *KernelArgs) {
+ assert(PM && "Runtime not initialized");
static_assert(std::is_convertible_v<TargetAsyncInfoTy, AsyncInfoTy>,
"Target AsyncInfoTy must be convertible to AsyncInfoTy.");
DP("Entering target region for device %" PRId64 " with entry point " DPxMOD
@@ -341,6 +352,7 @@ EXTERN int __tgt_activate_record_replay(int64_t DeviceId, uint64_t MemorySize,
void *VAddr, bool IsRecord,
bool SaveOutput,
uint64_t &ReqPtrArgOffset) {
+ assert(PM && "Runtime not initialized");
auto DeviceOrErr = PM->getDevice(DeviceId);
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
@@ -375,6 +387,7 @@ EXTERN int __tgt_target_kernel_replay(ident_t *Loc, int64_t DeviceId,
ptrdiff_t *TgtOffsets, int32_t NumArgs,
int32_t NumTeams, int32_t ThreadLimit,
uint64_t LoopTripCount) {
+ assert(PM && "Runtime not initialized");
if (checkDeviceAndCtors(DeviceId, Loc)) {
DP("Not offloading to device %" PRId64 "\n", DeviceId);
@@ -425,6 +438,8 @@ EXTERN void __tgt_push_mapper_component(void *RtMapperHandle, void *Base,
}
EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {
+ assert(PM && "Runtime not initialized");
+
std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();
InfoLevel.store(NewInfoLevel);
for (auto &R : PM->pluginAdaptors()) {
@@ -434,6 +449,8 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {
}
EXTERN int __tgt_print_device_info(int64_t DeviceId) {
+ assert(PM && "Runtime not initialized");
+
auto DeviceOrErr = PM->getDevice(DeviceId);
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
@@ -442,6 +459,8 @@ EXTERN int __tgt_print_device_info(int64_t DeviceId) {
}
EXTERN void __tgt_target_nowait_query(void **AsyncHandle) {
+ assert(PM && "Runtime not initialized");
+
if (!AsyncHandle || !*AsyncHandle) {
FATAL_MESSAGE0(
1, "Receive an invalid async handle from the current OpenMP task. Is "
diff --git a/openmp/libomptarget/test/offloading/runtime_init.c b/openmp/libomptarget/test/offloading/runtime_init.c
new file mode 100644
index 0000000000000..e1cd3da8919af
--- /dev/null
+++ b/openmp/libomptarget/test/offloading/runtime_init.c
@@ -0,0 +1,24 @@
+// RUN: %libomptarget-compile-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+extern void __tgt_rtl_init(void);
+extern void __tgt_rtl_deinit(void);
+
+// Sanity checks to make sure that this works and is thread safe.
+int main() {
+ __tgt_rtl_init();
+#pragma omp parallel num_threads(8)
+ {
+ __tgt_rtl_init();
+ __tgt_rtl_deinit();
+ }
+ __tgt_rtl_deinit();
+
+ __tgt_rtl_init();
+ __tgt_rtl_deinit();
+
+ // CHECK: PASS
+ printf("PASS\n");
+}
More information about the Openmp-commits
mailing list