[llvm-branch-commits] [llvm] [Offload][Omp] Ensure correct teardown order (PR #239629)
Alex Duran via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Nov 6 06:55:33 PST 2026
https://github.com/adurang created https://github.com/llvm/llvm-project/pull/239629
Use the new Offload API to ensure a proper libomptarget teardown
in case the process exits before reaching its `deinit` method.
Depending on different circumstances whether the shutdown happens
through `deinit` or through the liboffload `callback` everything
must be properly protected and allowed to be called safely multiple
times.
Assisted by Claude.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From e93df83b6cad273d6fe136759da82cf50fd456c7 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Fri, 6 Nov 2026 06:49:32 -0800
Subject: [PATCH] [Offload][Omp] Ensure correct teardown order
Use the new Offload API to ensure a proper libomptarget teardown
in case the process exits before reaching its `deinit` method.
Depending on different circumstances whether the shutdown happens
through `deinit` or through the liboffload `callback` everything
must be properly protected and allowed to be called safely multiple
times.
Assisted by Claude.
---
offload/include/PluginManager.h | 6 ++
offload/libompaccsupport/PluginManager.cpp | 15 +++-
offload/libomptarget/OffloadRTL.cpp | 75 +++++++++++++++----
offload/test/api/runtime_teardown.c | 21 ++++++
.../api/runtime_teardown_offload_disabled.c | 22 ++++++
.../test/api/runtime_teardown_shared_lib.c | 38 ++++++++++
6 files changed, 159 insertions(+), 18 deletions(-)
create mode 100644 offload/test/api/runtime_teardown.c
create mode 100644 offload/test/api/runtime_teardown_offload_disabled.c
create mode 100644 offload/test/api/runtime_teardown_shared_lib.c
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index b928365dc068a4..5855ab9b200d94 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -33,6 +33,7 @@
#include "llvm/Support/DynamicLibrary.h"
#include "llvm/Support/Error.h"
+#include <atomic>
#include <cstdint>
#include <list>
#include <memory>
@@ -54,8 +55,12 @@ struct PluginManager {
void init();
+ /// Deinitialize devices and release liboffload. Idempotent.
void deinit();
+ /// Deinitialize devices and drop platforms, without olShutDown. Idempotent.
+ void deinitDevices();
+
// Register a shared library with all (compatible) RTLs.
void registerLib(__tgt_bin_desc *Desc);
@@ -156,6 +161,7 @@ struct PluginManager {
bool RTLsLoaded = false;
// Whether init() called olInit, so deinit() must call olShutDown.
bool LibOffloadInitialized = false;
+ std::atomic<bool> DevicesDeinitialized{false};
llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
// List of all plugins, in use or not.
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index 7f3d781f60d344..b85842cbfa7af5 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -69,9 +69,9 @@ void PluginManager::init() {
ODBG(ODT_Init) << "RTLs loaded!";
}
-void PluginManager::deinit() {
- TIMESCOPE();
- ODBG(ODT_Deinit) << "Unloading RTLs...";
+void PluginManager::deinitDevices() {
+ if (DevicesDeinitialized.exchange(true))
+ return;
auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
for (DeviceTy &Device : devices(ExclusiveDevicesAccessor)) {
@@ -82,7 +82,16 @@ void PluginManager::deinit() {
Plugins.clear();
Platforms.clear();
+}
+
+void PluginManager::deinit() {
+ TIMESCOPE();
+ ODBG(ODT_Deinit) << "Unloading RTLs...";
+
+ deinitDevices();
+
if (LibOffloadInitialized) {
+ LibOffloadInitialized = false;
if (auto Err = olShutDown())
REPORT() << "Failed to denitialize liboffload: " << Err->Details;
}
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 6e20e5c3fcdf35..67443d356afdc1 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -25,7 +25,17 @@ static std::mutex &getPluginMutex() {
static std::mutex Mutex;
return Mutex;
}
+/// Guards PM and TornDown against the shutdown callback.
+static std::mutex &getTeardownMutex() {
+ static std::mutex Mutex;
+ return Mutex;
+}
static uint32_t RefCount = 0;
+static bool TornDown = false;
+// A shutdown callback is pending in liboffload; register at most one.
+static bool ShutdownCallbackPending = false;
+static void teardownRuntime();
+static void shutdownCallback(void *);
std::atomic<bool> RTLAlive{false};
std::atomic<int> RTLOngoingSyncs{0};
OmpPluginManager *PM = nullptr;
@@ -39,6 +49,38 @@ static void checkRuntimeEnvironment() {
ShmemEnvarName);
}
+/// Idempotent teardown, called from deinitRuntime() and the shutdown callback.
+/// Must not call olShutDown or take the plugin mutex.
+static void teardownRuntimeLocked() {
+ if (!PM || TornDown)
+ return;
+ TornDown = true;
+
+ ODBG(ODT_Deinit) << "Tearing down offload library!";
+ // RTL deinitialization has started
+ RTLAlive = false;
+ while (RTLOngoingSyncs > 0) {
+ ODBG(ODT_Sync) << "Waiting for ongoing syncs to finish, count:"
+ << RTLOngoingSyncs.load();
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ }
+ // Must precede device deinit: the backends may be unloaded afterwards.
+ PM->InteropTbl.clear();
+ PM->deinitDevices();
+}
+
+static void teardownRuntime() {
+ std::scoped_lock<std::mutex> Lock(getTeardownMutex());
+ teardownRuntimeLocked();
+}
+
+static void shutdownCallback(void *) {
+ std::scoped_lock<std::mutex> Lock(getTeardownMutex());
+ // liboffload drops the callback once it ran.
+ ShutdownCallbackPending = false;
+ teardownRuntimeLocked();
+}
+
void initRuntime() {
std::scoped_lock<std::mutex> Lock(getPluginMutex());
Profiler::get();
@@ -49,7 +91,11 @@ void initRuntime() {
RefCount++;
if (RefCount == 1) {
assert(!PM && "Runtime already initialized");
- PM = new llvm::omp::target::OmpPluginManager();
+ {
+ std::scoped_lock<std::mutex> TeardownLock(getTeardownMutex());
+ PM = new llvm::omp::target::OmpPluginManager();
+ TornDown = false;
+ }
ODBG(ODT_Init) << "Init offload library!";
#ifdef OMPT_SUPPORT
@@ -62,13 +108,17 @@ void initRuntime() {
else
ODBG(ODT_Init) << "Offload is disabled. Skipping plugin initialization";
PM->registerDelayedLibraries();
- // After all plugins are initialized, register atExit cleanup handlers.
- std::atexit([]() {
- // Interop cleanup should be done before the plugins are deinitialized as
- // the backend libraries may be already unloaded.
- if (PM)
- PM->InteropTbl.clear();
- });
+ // Tear down before liboffload deinitializes the platforms at exit.
+ if (PM->isLibOffloadInitialized()) {
+ std::scoped_lock<std::mutex> TeardownLock(getTeardownMutex());
+ if (!ShutdownCallbackPending) {
+ if (ol_result_t Res =
+ olRegisterShutdownCallback(shutdownCallback, nullptr))
+ REPORT() << "Failed to register shutdown callback: " << Res->Details;
+ else
+ ShutdownCallbackPending = true;
+ }
+ }
// RTL initialization is complete
RTLAlive = true;
@@ -81,14 +131,9 @@ void deinitRuntime() {
if (RefCount == 1) {
ODBG(ODT_Deinit) << "Deinit offload library!";
- // RTL deinitialization has started
- RTLAlive = false;
- while (RTLOngoingSyncs > 0) {
- ODBG(ODT_Sync) << "Waiting for ongoing syncs to finish, count:"
- << RTLOngoingSyncs.load();
- std::this_thread::sleep_for(std::chrono::milliseconds(100));
- }
+ teardownRuntime();
PM->deinit();
+ std::scoped_lock<std::mutex> TeardownLock(getTeardownMutex());
delete PM;
PM = nullptr;
}
diff --git a/offload/test/api/runtime_teardown.c b/offload/test/api/runtime_teardown.c
new file mode 100644
index 00000000000000..00850fbf1d043f
--- /dev/null
+++ b/offload/test/api/runtime_teardown.c
@@ -0,0 +1,21 @@
+// RUN: %libomptarget-compile-generic
+// RUN: env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 \
+// RUN: | %fcheck-generic
+
+// The runtime is torn down exactly once at exit.
+
+// REQUIRES: libomptarget-debug
+
+#include <stdio.h>
+
+int main() {
+ int X = 0;
+#pragma omp target map(tofrom : X)
+ X = 1;
+ printf("X = %d\n", X);
+ return X != 1;
+}
+
+// CHECK-DAG: X = 1
+// CHECK-DAG: Tearing down offload library!
+// CHECK-NOT: Tearing down offload library!
diff --git a/offload/test/api/runtime_teardown_offload_disabled.c b/offload/test/api/runtime_teardown_offload_disabled.c
new file mode 100644
index 00000000000000..85184ae71c6ea8
--- /dev/null
+++ b/offload/test/api/runtime_teardown_offload_disabled.c
@@ -0,0 +1,22 @@
+// RUN: %libomptarget-compile-generic
+// RUN: env OMP_TARGET_OFFLOAD=disabled LIBOMPTARGET_DEBUG=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// Offload disabled: no shutdown callback, teardown must still happen.
+
+// REQUIRES: libomptarget-debug
+
+#include <stdio.h>
+
+int main() {
+ int X = 0;
+#pragma omp target map(tofrom : X)
+ X = 1;
+ printf("X = %d\n", X);
+ return X != 1;
+}
+
+// CHECK-DAG: Offload is disabled. Skipping plugin initialization
+// CHECK-DAG: X = 1
+// CHECK-DAG: Tearing down offload library!
+// CHECK-NOT: Tearing down offload library!
diff --git a/offload/test/api/runtime_teardown_shared_lib.c b/offload/test/api/runtime_teardown_shared_lib.c
new file mode 100644
index 00000000000000..26c280471b45d9
--- /dev/null
+++ b/offload/test/api/runtime_teardown_shared_lib.c
@@ -0,0 +1,38 @@
+// RUN: rm -rf %t.testdir && mkdir -p %t.testdir
+// RUN: %clang-generic -DSHARED_LIB -fPIC -shared %s \
+// RUN: -o %t.testdir/libompshared.so
+// RUN: %clang %s -Wl,-rpath,%t.testdir -L%t.testdir -lompshared -o %t
+// RUN: env LIBOMPTARGET_DEBUG=1 %t 2>&1 | %fcheck-generic
+
+// OpenMP only in the shared library; the runtime is torn down exactly once.
+
+// REQUIRES: libomptarget-debug
+
+#ifdef SHARED_LIB
+
+// Library, built with OpenMP offloading.
+int run_on_device(int V) {
+ int X = V;
+#pragma omp target map(tofrom : X)
+ X += 1;
+ return X;
+}
+
+#else
+
+// Executable, built without OpenMP.
+#include <stdio.h>
+
+int run_on_device(int V);
+
+int main() {
+ int R = run_on_device(41);
+ printf("R = %d\n", R);
+ return R != 42;
+}
+
+#endif
+
+// CHECK-DAG: R = 42
+// CHECK-DAG: Tearing down offload library!
+// CHECK-NOT: Tearing down offload library!
More information about the llvm-branch-commits
mailing list