[llvm] [offload] Extract OMPT out of PluginInterface (PR #221726)
Ivan R. Ivanov via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 06:23:24 PDT 2026
https://github.com/ivanradanov created https://github.com/llvm/llvm-project/pull/221726
Some of the OMPT callbacks, namely the device lifetime and image related ones, lived in the generic PluginInterface even though they are OpenMP specific. This patch extracts them out to the PluginManager, and removes some device ID tracking functionality which was only necessary for OpenMP from PluginInterface. The OpenMP device ID mapping should live only in libomptarget.
We explicitly deinit each device while calling its finalize callback prior to that, before deinitting the plugins. The plugin deinit also deinits the devices, but this is a no-op for already deinitted devices so there are no problems.
This should give us a clean slate to implement a generic callback interface for liboffload that can implement the needs of various offload languages (e.g. OpenMP and OpenACC) if deemed necessary and if deemed it belongs in liboffload in the first place.
>From 5294f54fdd44b358af73595d97e427d7ecc7eea5 Mon Sep 17 00:00:00 2001
From: Ivan Radanov Ivanov <iivanov at nvidia.com>
Date: Fri, 4 Sep 2026 09:57:33 -0700
Subject: [PATCH 1/2] pull ompt out of PluginInterface
---
offload/include/OpenMP/OMPT/Callback.h | 3 +-
offload/include/OpenMP/OMPT/Interface.h | 3 +
offload/liboffload/src/OffloadImpl.cpp | 15 -----
offload/libompaccsupport/PluginManager.cpp | 16 +++--
offload/libompaccsupport/device.cpp | 16 +++++
offload/libomptarget/OffloadRTL.cpp | 3 -
offload/libomptarget/OpenMP/OMPT/Callback.cpp | 6 +-
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 1 -
.../common/include/PluginInterface.h | 26 --------
.../common/src/PluginInterface.cpp | 60 -------------------
offload/plugins-nextgen/cuda/src/rtl.cpp | 1 -
offload/plugins-nextgen/host/src/rtl.cpp | 1 -
12 files changed, 33 insertions(+), 118 deletions(-)
diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 2bc3f59dcd3ee..c2ea9292ecb36 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -50,8 +50,7 @@ namespace target {
namespace ompt {
#define declareOmptCallback(Name, Type, Code) extern Name##_t Name##_fn;
-FOREACH_OMPT_NOEMI_EVENT(declareOmptCallback)
-FOREACH_OMPT_EMI_EVENT(declareOmptCallback)
+FOREACH_OMPT_TARGET_CALLBACK(declareOmptCallback)
#undef declareOmptCallback
/// This function will call an OpenMP API function. Which in turn will lookup a
diff --git a/offload/include/OpenMP/OMPT/Interface.h b/offload/include/OpenMP/OMPT/Interface.h
index 6961641769b76..3ee3342fbe9eb 100644
--- a/offload/include/OpenMP/OMPT/Interface.h
+++ b/offload/include/OpenMP/OMPT/Interface.h
@@ -24,6 +24,8 @@
#include <tuple>
#define OMPT_IF_BUILT(stmt) stmt
+#define OMPT_IF_BUILT_AND_INITIALIZED(stmt) \
+ OMPT_IF_BUILT(performIfOmptInitialized(stmt))
/// Callbacks for target regions require task_data representing the
/// encountering task.
@@ -337,6 +339,7 @@ class ReturnAddressSetterRAII {
#else
#define OMPT_IF_BUILT(stmt)
+#define OMPT_IF_BUILT_AND_INITIALIZED(stmt)
#endif
#endif // OFFLOAD_INCLUDE_OPENMP_OMPT_INTERFACE_H
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 11c07483b38aa..50d5682bf1157 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -21,21 +21,6 @@
#include <cstdint>
#include <mutex>
-// TODO: Some plugins expect to be linked into libomptarget which defines these
-// symbols to implement ompt callbacks. The least invasive workaround here is to
-// define them in libLLVMOffload as false/null so they are never used. In future
-// it would be better to allow the plugins to implement callbacks without
-// pulling in details from libomptarget.
-#ifdef OMPT_SUPPORT
-namespace llvm::omp::target {
-namespace ompt {
-bool Initialized = false;
-ompt_get_callback_t lookupCallbackByCode = nullptr;
-ompt_function_lookup_t lookupCallbackByName = nullptr;
-} // namespace ompt
-} // namespace llvm::omp::target
-#endif
-
using namespace llvm::omp::target;
using namespace llvm::omp::target::plugin;
using namespace error;
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index 41b653a60adfd..5f67a204f4203 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -12,6 +12,7 @@
#include "PluginManager.h"
#include "OffloadPolicy.h"
+#include "OpenMP/OMPT/Interface.h"
#include "Shared/Debug.h"
#include "Shared/Profile.h"
#include "device.h"
@@ -20,6 +21,10 @@
#include "llvm/Support/ErrorHandling.h"
#include <memory>
+#ifdef OMPT_SUPPORT
+using namespace llvm::omp::target::ompt;
+#endif
+
using namespace llvm;
using namespace llvm::sys;
using namespace llvm::omp::target::debug;
@@ -54,6 +59,12 @@ void PluginManager::deinit() {
TIMESCOPE();
ODBG(ODT_Deinit) << "Unloading RTLs...";
+ OMPT_IF_BUILT_AND_INITIALIZED({
+ auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
+ for (DeviceTy &Device : devices(ExclusiveDevicesAccessor))
+ performOmptCallback(device_finalize, Device.DeviceID);
+ });
+
for (auto &Plugin : Plugins) {
if (!Plugin->is_initialized())
continue;
@@ -99,11 +110,6 @@ bool PluginManager::initializeDevice(GenericPluginTy &Plugin,
int32_t UserId = ExclusiveDevicesAccessor->size();
- // Set the device identifier offset in the plugin.
-#ifdef OMPT_SUPPORT
- Plugin.set_device_identifier(UserId, DeviceId);
-#endif
-
auto Device = std::make_unique<DeviceTy>(&Plugin, UserId, DeviceId);
if (auto Err = Device->init()) {
std::string InfoMsg = toString(std::move(Err));
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 688746477861c..5aec1156930d6 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -87,6 +87,14 @@ llvm::Error DeviceTy::init() {
"failed to initialize device %d\n",
DeviceID);
+ OMPT_IF_BUILT_AND_INITIALIZED({
+ GenericDeviceTy &GenericDevice = RTL->getDevice(RTLDeviceID);
+ std::string ComputeUnitKind = GenericDevice.getComputeUnitKind();
+ performOmptCallback(device_initialize, DeviceID, ComputeUnitKind.c_str(),
+ reinterpret_cast<ompt_device_t *>(&GenericDevice),
+ lookupCallbackByName, /*documentation=*/nullptr);
+ });
+
// Enables recording kernels if set.
BoolEnvar OMPX_RecordKernel("LIBOMPTARGET_RECORD", false);
if (OMPX_RecordKernel) {
@@ -220,6 +228,14 @@ DeviceTy::loadBinary(__tgt_device_image *Img) {
return error::createOffloadError(error::ErrorCode::INVALID_BINARY,
"failed to load binary %p", Img);
+ OMPT_IF_BUILT_AND_INITIALIZED(performOmptCallback(
+ device_load, DeviceID, /*FileName=*/nullptr, /*FileOffset=*/0,
+ /*VmaInFile=*/nullptr,
+ reinterpret_cast<uintptr_t>(Img->ImageEnd) -
+ reinterpret_cast<uintptr_t>(Img->ImageStart),
+ const_cast<void *>(Img->ImageStart),
+ /*DeviceAddr=*/nullptr, /*ModuleId=*/0));
+
// This symbol is optional.
void *DeviceEnvironmentPtr;
if (RTL->get_global(Binary, sizeof(DeviceEnvironmentTy),
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 9b02376609cee..96e4027dfc99f 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -16,9 +16,6 @@
#include "Shared/Debug.h"
#include "Shared/Profile.h"
-#ifdef OMPT_SUPPORT
-extern void llvm::omp::target::ompt::connectLibrary();
-#endif
using namespace llvm::omp::target::debug;
static std::mutex PluginMtx;
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 150fcb85015f7..f532986ce07a0 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -31,8 +31,7 @@ extern "C" void ompt_libomp_connect(ompt_start_tool_result_t *);
// Define OMPT callback functions (bound to actual callbacks later on)
#define defineOmptCallback(Name, Type, Code) \
Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
-FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback)
-FOREACH_OMPT_EMI_EVENT(defineOmptCallback)
+FOREACH_OMPT_TARGET_CALLBACK(defineOmptCallback)
#undef defineOmptCallback
// See definition in OpenMP (omp.h.var/omp_lib.(F90|h).var)
@@ -552,8 +551,7 @@ void llvm::omp::target::ompt::connectLibrary() {
lookupCallbackByCode( \
(ompt_callbacks_t)(Code), \
(ompt_callback_t *)&(llvm::omp::target::ompt::Name##_fn));
- FOREACH_OMPT_NOEMI_EVENT(bindOmptCallback)
- FOREACH_OMPT_EMI_EVENT(bindOmptCallback)
+ FOREACH_OMPT_TARGET_CALLBACK(bindOmptCallback)
#undef bindOmptCallback
ODBG(ODT_Tool) << "Exiting connectLibrary";
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 281b9e3795a54..bfedf163946d4 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -32,7 +32,6 @@
#include "GlobalHandler.h"
#include "OffloadAPI.h"
-#include "OpenMP/OMPT/Callback.h"
#include "PluginInterface.h"
#include "UtilitiesRTL.h"
#include "omptarget.h"
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 29513661867b1..3e2cc0c4647c9 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -37,10 +37,6 @@
#include "RecordReplay.h"
#include "omptarget.h"
-#ifdef OMPT_SUPPORT
-#include "omp-tools.h"
-#endif
-
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/Hashing.h"
@@ -1511,16 +1507,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
/// This is used to run the RPC server during task synchronization.
RPCServerTy *RPCServer;
-#ifdef OMPT_SUPPORT
- /// OMPT callback functions
-#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
- FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
-#undef defineOmptCallback
-
- /// Internal representation for OMPT device (initialize & finalize)
- std::atomic<bool> OmptInitialized;
-#endif
-
/// The total per-block native shared memory that a kernel may use.
size_t MaxBlockSharedMemSize = 0;
};
@@ -1573,12 +1559,6 @@ struct GenericPluginTy {
/// Get the number of active devices.
int32_t getNumDevices() const { return NumDevices; }
- /// Get the plugin-specific device identifier.
- int32_t getUserId(int32_t DeviceId) const {
- assert(UserDeviceIds.contains(DeviceId) && "No user-id registered");
- return UserDeviceIds.at(DeviceId);
- }
-
/// Get the UID for the host device.
static constexpr const char *getHostDeviceUid() { return "HOST"; }
@@ -1818,9 +1798,6 @@ struct GenericPluginTy {
/// Remove the event from the plugin.
void set_info_flag(uint32_t NewInfoLevel);
- /// Sets the offset into the devices for use by OMPT.
- int32_t set_device_identifier(int32_t UserId, int32_t DeviceId);
-
/// Returns if the plugin can support automatic copy.
int32_t use_auto_zero_copy(int32_t DeviceId);
@@ -1876,9 +1853,6 @@ struct GenericPluginTy {
/// Number of devices available for the plugin.
int32_t NumDevices = 0;
- /// Map of plugin device identifiers to the user device identifier.
- llvm::DenseMap<int32_t, int32_t> UserDeviceIds;
-
/// Array of pointers to the devices. Initially, they are all set to nullptr.
/// Once a device is initialized, the pointer is stored in the position given
/// by its device id. A position with nullptr means that the corresponding
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 573369dc3ba57..595cfdd333d87 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -21,11 +21,6 @@
#include "Utils/ELF.h"
#include "omptarget.h"
-#ifdef OMPT_SUPPORT
-#include "OpenMP/OMPT/Callback.h"
-#include "omp-tools.h"
-#endif
-
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/Support/Error.h"
@@ -467,22 +462,6 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
// vendor (u)uid will become available later.
setDeviceUidFromVendorUid(std::to_string(static_cast<uint64_t>(DeviceId)));
-#ifdef OMPT_SUPPORT
- OmptInitialized.store(false);
- // Bind the callbacks to this device's member functions
-#define bindOmptCallback(Name, Type, Code) \
- if (ompt::Initialized && ompt::lookupCallbackByCode) { \
- ompt::lookupCallbackByCode((ompt_callbacks_t)(Code), \
- ((ompt_callback_t *)&(Name##_fn))); \
- ODBG(OLDT_Tool) << "OMPT: class bound " << #Name << "=" \
- << ((void *)(uint64_t)Name##_fn); \
- }
-
- FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
-#undef bindOmptCallback
-
-#endif
-
// Envar that indicates whether mapped host buffers should be locked
// automatically. The possible values are boolean (on/off) and a special:
// off: Mapped host buffers are not locked.
@@ -514,18 +493,6 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
if (auto Err = initImpl(Plugin))
return Err;
-#ifdef OMPT_SUPPORT
- if (ompt::Initialized) {
- bool ExpectedStatus = false;
- if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
- performOmptCallback(device_initialize, Plugin.getUserId(DeviceId),
- /*type=*/getComputeUnitKind().c_str(),
- /*device=*/reinterpret_cast<ompt_device_t *>(this),
- /*lookup=*/ompt::lookupCallbackByName,
- /*documentation=*/nullptr);
- }
-#endif
-
// Read and reinitialize the envars that depend on the device initialization.
// Notice these two envars may change the stack size and heap size of the
// device, so they need the device properly initialized.
@@ -635,14 +602,6 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
RecordReplay = nullptr;
}
-#ifdef OMPT_SUPPORT
- if (ompt::Initialized) {
- bool ExpectedStatus = true;
- if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
- performOmptCallback(device_finalize, Plugin.getUserId(DeviceId));
- }
-#endif
-
return deinitImpl();
}
Expected<DeviceImageTy *>
@@ -690,18 +649,6 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin, StringRef InputTgtImage,
if (auto Err = setupRPCServer(Plugin, *Image))
return std::move(Err);
-#ifdef OMPT_SUPPORT
- if (ompt::Initialized) {
- size_t Bytes = InputTgtImage.size();
- performOmptCallback(
- device_load, Plugin.getUserId(DeviceId),
- /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
- /*ImgSize=*/Bytes,
- /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
- /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
- }
-#endif
-
// Call any global constructors present on the device.
if (auto Err = callGlobalConstructors(Plugin, *Image))
return std::move(Err);
@@ -1840,13 +1787,6 @@ void GenericPluginTy::set_info_flag(uint32_t NewInfoLevel) {
InfoLevel.store(NewInfoLevel);
}
-int32_t GenericPluginTy::set_device_identifier(int32_t UserId,
- int32_t DeviceId) {
- UserDeviceIds[DeviceId] = UserId;
-
- return OFFLOAD_SUCCESS;
-}
-
int32_t GenericPluginTy::use_auto_zero_copy(int32_t DeviceId) {
return getDevice(DeviceId).useAutoZeroCopy();
}
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 72e5dcf115fe9..0efedf2b8bdc6 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -22,7 +22,6 @@
#include "GlobalHandler.h"
#include "OffloadAPI.h"
-#include "OpenMP/OMPT/Callback.h"
#include "PluginInterface.h"
#include "Utils/ELF.h"
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 55ada2f82c360..37114dec5ef66 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -21,7 +21,6 @@
#include "GlobalHandler.h"
#include "OffloadAPI.h"
-#include "OpenMP/OMPT/Callback.h"
#include "PluginInterface.h"
#include "omptarget.h"
>From 33a929beb5f8f2741c2102d000a22d620070e483 Mon Sep 17 00:00:00 2001
From: Ivan Radanov Ivanov <iivanov at nvidia.com>
Date: Mon, 7 Sep 2026 06:09:41 -0700
Subject: [PATCH 2/2] call deinit per device
---
offload/include/device.h | 6 +++---
offload/libompaccsupport/PluginManager.cpp | 11 +++--------
offload/libompaccsupport/device.cpp | 10 ++++++++++
3 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/offload/include/device.h b/offload/include/device.h
index 266a2a675df0c..4c871c94ef058 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -59,6 +59,9 @@ struct DeviceTy {
/// Try to initialize the device and return any failure.
llvm::Error init();
+ /// Deinitialize the plugin device associated with this DeviceTy.
+ void deinit();
+
/// Provide access to the mapping handler.
MappingInfoTy &getMappingInfo() { return MappingInfo; }
@@ -185,9 +188,6 @@ struct DeviceTy {
}
private:
- /// Deinitialize the device (and plugin).
- void deinit();
-
/// All offload entries available on this device.
using DeviceOffloadEntriesMapTy =
llvm::DenseMap<llvm::StringRef, OffloadEntryTy>;
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index 5f67a204f4203..f0ae6fcb90cfc 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -12,7 +12,6 @@
#include "PluginManager.h"
#include "OffloadPolicy.h"
-#include "OpenMP/OMPT/Interface.h"
#include "Shared/Debug.h"
#include "Shared/Profile.h"
#include "device.h"
@@ -21,10 +20,6 @@
#include "llvm/Support/ErrorHandling.h"
#include <memory>
-#ifdef OMPT_SUPPORT
-using namespace llvm::omp::target::ompt;
-#endif
-
using namespace llvm;
using namespace llvm::sys;
using namespace llvm::omp::target::debug;
@@ -59,11 +54,11 @@ void PluginManager::deinit() {
TIMESCOPE();
ODBG(ODT_Deinit) << "Unloading RTLs...";
- OMPT_IF_BUILT_AND_INITIALIZED({
+ {
auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
for (DeviceTy &Device : devices(ExclusiveDevicesAccessor))
- performOmptCallback(device_finalize, Device.DeviceID);
- });
+ Device.deinit();
+ }
for (auto &Plugin : Plugins) {
if (!Plugin->is_initialized())
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 5aec1156930d6..d803dc2aa9193 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -127,6 +127,16 @@ llvm::Error DeviceTy::init() {
return llvm::Error::success();
}
+void DeviceTy::deinit() {
+ OMPT_IF_BUILT_AND_INITIALIZED(performOmptCallback(device_finalize, DeviceID));
+
+ if (auto Err = RTL->deinitDevice(RTLDeviceID)) {
+ std::string InfoMsg = toString(std::move(Err));
+ ODBG(ODT_Deinit) << "Failed to deinit device " << DeviceID << ": "
+ << InfoMsg;
+ }
+}
+
// Extract the mapping of host function pointers to device function pointers
// from the entry table. Functions marked as 'indirect' in OpenMP will have
// offloading entries generated for them which map the host's function pointer
More information about the llvm-commits
mailing list