[llvm-branch-commits] [llvm] [Offload][Omp] Ensure correct teardown order (PR #239629)

Alex Duran via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Fri Nov 6 06:55:33 PST 2026


https://github.com/adurang created https://github.com/llvm/llvm-project/pull/239629

Use the new Offload API to ensure a proper libomptarget teardown
in case the process exits before reaching its `deinit` method.

Depending on different circumstances whether the shutdown happens
through `deinit` or through the liboffload `callback` everything
must be properly protected and allowed to be called safely multiple
times.

Assisted by Claude.

---

<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>

>From e93df83b6cad273d6fe136759da82cf50fd456c7 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Fri, 6 Nov 2026 06:49:32 -0800
Subject: [PATCH] [Offload][Omp] Ensure correct teardown order

Use the new Offload API to ensure a proper libomptarget teardown
in case the process exits before reaching its `deinit` method.

Depending on different circumstances whether the shutdown happens
through `deinit` or through the liboffload `callback` everything
must be properly protected and allowed to be called safely multiple
times.

Assisted by Claude.
---
 offload/include/PluginManager.h               |  6 ++
 offload/libompaccsupport/PluginManager.cpp    | 15 +++-
 offload/libomptarget/OffloadRTL.cpp           | 75 +++++++++++++++----
 offload/test/api/runtime_teardown.c           | 21 ++++++
 .../api/runtime_teardown_offload_disabled.c   | 22 ++++++
 .../test/api/runtime_teardown_shared_lib.c    | 38 ++++++++++
 6 files changed, 159 insertions(+), 18 deletions(-)
 create mode 100644 offload/test/api/runtime_teardown.c
 create mode 100644 offload/test/api/runtime_teardown_offload_disabled.c
 create mode 100644 offload/test/api/runtime_teardown_shared_lib.c

diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index b928365dc068a4..5855ab9b200d94 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -33,6 +33,7 @@
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Error.h"
 
+#include <atomic>
 #include <cstdint>
 #include <list>
 #include <memory>
@@ -54,8 +55,12 @@ struct PluginManager {
 
   void init();
 
+  /// Deinitialize devices and release liboffload. Idempotent.
   void deinit();
 
+  /// Deinitialize devices and drop platforms, without olShutDown. Idempotent.
+  void deinitDevices();
+
   // Register a shared library with all (compatible) RTLs.
   void registerLib(__tgt_bin_desc *Desc);
 
@@ -156,6 +161,7 @@ struct PluginManager {
   bool RTLsLoaded = false;
   // Whether init() called olInit, so deinit() must call olShutDown.
   bool LibOffloadInitialized = false;
+  std::atomic<bool> DevicesDeinitialized{false};
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
 
   // List of all plugins, in use or not.
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index 7f3d781f60d344..b85842cbfa7af5 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -69,9 +69,9 @@ void PluginManager::init() {
   ODBG(ODT_Init) << "RTLs loaded!";
 }
 
-void PluginManager::deinit() {
-  TIMESCOPE();
-  ODBG(ODT_Deinit) << "Unloading RTLs...";
+void PluginManager::deinitDevices() {
+  if (DevicesDeinitialized.exchange(true))
+    return;
 
   auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
   for (DeviceTy &Device : devices(ExclusiveDevicesAccessor)) {
@@ -82,7 +82,16 @@ void PluginManager::deinit() {
 
   Plugins.clear();
   Platforms.clear();
+}
+
+void PluginManager::deinit() {
+  TIMESCOPE();
+  ODBG(ODT_Deinit) << "Unloading RTLs...";
+
+  deinitDevices();
+
   if (LibOffloadInitialized) {
+    LibOffloadInitialized = false;
     if (auto Err = olShutDown())
       REPORT() << "Failed to denitialize liboffload: " << Err->Details;
   }
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 6e20e5c3fcdf35..67443d356afdc1 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -25,7 +25,17 @@ static std::mutex &getPluginMutex() {
   static std::mutex Mutex;
   return Mutex;
 }
+/// Guards PM and TornDown against the shutdown callback.
+static std::mutex &getTeardownMutex() {
+  static std::mutex Mutex;
+  return Mutex;
+}
 static uint32_t RefCount = 0;
+static bool TornDown = false;
+// A shutdown callback is pending in liboffload; register at most one.
+static bool ShutdownCallbackPending = false;
+static void teardownRuntime();
+static void shutdownCallback(void *);
 std::atomic<bool> RTLAlive{false};
 std::atomic<int> RTLOngoingSyncs{0};
 OmpPluginManager *PM = nullptr;
@@ -39,6 +49,38 @@ static void checkRuntimeEnvironment() {
             ShmemEnvarName);
 }
 
+/// Idempotent teardown, called from deinitRuntime() and the shutdown callback.
+/// Must not call olShutDown or take the plugin mutex.
+static void teardownRuntimeLocked() {
+  if (!PM || TornDown)
+    return;
+  TornDown = true;
+
+  ODBG(ODT_Deinit) << "Tearing down offload library!";
+  // RTL deinitialization has started
+  RTLAlive = false;
+  while (RTLOngoingSyncs > 0) {
+    ODBG(ODT_Sync) << "Waiting for ongoing syncs to finish, count:"
+                   << RTLOngoingSyncs.load();
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  }
+  // Must precede device deinit: the backends may be unloaded afterwards.
+  PM->InteropTbl.clear();
+  PM->deinitDevices();
+}
+
+static void teardownRuntime() {
+  std::scoped_lock<std::mutex> Lock(getTeardownMutex());
+  teardownRuntimeLocked();
+}
+
+static void shutdownCallback(void *) {
+  std::scoped_lock<std::mutex> Lock(getTeardownMutex());
+  // liboffload drops the callback once it ran.
+  ShutdownCallbackPending = false;
+  teardownRuntimeLocked();
+}
+
 void initRuntime() {
   std::scoped_lock<std::mutex> Lock(getPluginMutex());
   Profiler::get();
@@ -49,7 +91,11 @@ void initRuntime() {
   RefCount++;
   if (RefCount == 1) {
     assert(!PM && "Runtime already initialized");
-    PM = new llvm::omp::target::OmpPluginManager();
+    {
+      std::scoped_lock<std::mutex> TeardownLock(getTeardownMutex());
+      PM = new llvm::omp::target::OmpPluginManager();
+      TornDown = false;
+    }
 
     ODBG(ODT_Init) << "Init offload library!";
 #ifdef OMPT_SUPPORT
@@ -62,13 +108,17 @@ void initRuntime() {
     else
       ODBG(ODT_Init) << "Offload is disabled. Skipping plugin initialization";
     PM->registerDelayedLibraries();
-    // After all plugins are initialized, register atExit cleanup handlers.
-    std::atexit([]() {
-      // Interop cleanup should be done before the plugins are deinitialized as
-      // the backend libraries may be already unloaded.
-      if (PM)
-        PM->InteropTbl.clear();
-    });
+    // Tear down before liboffload deinitializes the platforms at exit.
+    if (PM->isLibOffloadInitialized()) {
+      std::scoped_lock<std::mutex> TeardownLock(getTeardownMutex());
+      if (!ShutdownCallbackPending) {
+        if (ol_result_t Res =
+                olRegisterShutdownCallback(shutdownCallback, nullptr))
+          REPORT() << "Failed to register shutdown callback: " << Res->Details;
+        else
+          ShutdownCallbackPending = true;
+      }
+    }
 
     // RTL initialization is complete
     RTLAlive = true;
@@ -81,14 +131,9 @@ void deinitRuntime() {
 
   if (RefCount == 1) {
     ODBG(ODT_Deinit) << "Deinit offload library!";
-    // RTL deinitialization has started
-    RTLAlive = false;
-    while (RTLOngoingSyncs > 0) {
-      ODBG(ODT_Sync) << "Waiting for ongoing syncs to finish, count:"
-                     << RTLOngoingSyncs.load();
-      std::this_thread::sleep_for(std::chrono::milliseconds(100));
-    }
+    teardownRuntime();
     PM->deinit();
+    std::scoped_lock<std::mutex> TeardownLock(getTeardownMutex());
     delete PM;
     PM = nullptr;
   }
diff --git a/offload/test/api/runtime_teardown.c b/offload/test/api/runtime_teardown.c
new file mode 100644
index 00000000000000..00850fbf1d043f
--- /dev/null
+++ b/offload/test/api/runtime_teardown.c
@@ -0,0 +1,21 @@
+// RUN: %libomptarget-compile-generic
+// RUN: env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 \
+// RUN:   | %fcheck-generic
+
+// The runtime is torn down exactly once at exit.
+
+// REQUIRES: libomptarget-debug
+
+#include <stdio.h>
+
+int main() {
+  int X = 0;
+#pragma omp target map(tofrom : X)
+  X = 1;
+  printf("X = %d\n", X);
+  return X != 1;
+}
+
+// CHECK-DAG: X = 1
+// CHECK-DAG: Tearing down offload library!
+// CHECK-NOT: Tearing down offload library!
diff --git a/offload/test/api/runtime_teardown_offload_disabled.c b/offload/test/api/runtime_teardown_offload_disabled.c
new file mode 100644
index 00000000000000..85184ae71c6ea8
--- /dev/null
+++ b/offload/test/api/runtime_teardown_offload_disabled.c
@@ -0,0 +1,22 @@
+// RUN: %libomptarget-compile-generic
+// RUN: env OMP_TARGET_OFFLOAD=disabled LIBOMPTARGET_DEBUG=1 \
+// RUN:   %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// Offload disabled: no shutdown callback, teardown must still happen.
+
+// REQUIRES: libomptarget-debug
+
+#include <stdio.h>
+
+int main() {
+  int X = 0;
+#pragma omp target map(tofrom : X)
+  X = 1;
+  printf("X = %d\n", X);
+  return X != 1;
+}
+
+// CHECK-DAG: Offload is disabled. Skipping plugin initialization
+// CHECK-DAG: X = 1
+// CHECK-DAG: Tearing down offload library!
+// CHECK-NOT: Tearing down offload library!
diff --git a/offload/test/api/runtime_teardown_shared_lib.c b/offload/test/api/runtime_teardown_shared_lib.c
new file mode 100644
index 00000000000000..26c280471b45d9
--- /dev/null
+++ b/offload/test/api/runtime_teardown_shared_lib.c
@@ -0,0 +1,38 @@
+// RUN: rm -rf %t.testdir && mkdir -p %t.testdir
+// RUN: %clang-generic -DSHARED_LIB -fPIC -shared %s \
+// RUN:   -o %t.testdir/libompshared.so
+// RUN: %clang %s -Wl,-rpath,%t.testdir -L%t.testdir -lompshared -o %t
+// RUN: env LIBOMPTARGET_DEBUG=1 %t 2>&1 | %fcheck-generic
+
+// OpenMP only in the shared library; the runtime is torn down exactly once.
+
+// REQUIRES: libomptarget-debug
+
+#ifdef SHARED_LIB
+
+// Library, built with OpenMP offloading.
+int run_on_device(int V) {
+  int X = V;
+#pragma omp target map(tofrom : X)
+  X += 1;
+  return X;
+}
+
+#else
+
+// Executable, built without OpenMP.
+#include <stdio.h>
+
+int run_on_device(int V);
+
+int main() {
+  int R = run_on_device(41);
+  printf("R = %d\n", R);
+  return R != 42;
+}
+
+#endif
+
+// CHECK-DAG: R = 42
+// CHECK-DAG: Tearing down offload library!
+// CHECK-NOT: Tearing down offload library!



More information about the llvm-branch-commits mailing list