[Openmp-commits] [llvm] [openmp] [OMPT][OpenMP] Use omp_initial_device for host in callbacks (PR #192924)
via Openmp-commits
openmp-commits at lists.llvm.org
Mon Apr 20 02:22:10 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-offload
Author: Jan André Reuter (Thyre)
<details>
<summary>Changes</summary>
The OpenMP specification offers different ways for identifying the host device. While users of the OpenMP API can use `omp_get_initial_device()` or the constant `omp_initial_device` (available since OpenMP v5.2), a tool needs to rely on the `initial_device_num` passed by the OpenMP runtime during the `initialize` callback.
In #<!-- -->134451, it was discovered that the `initial_device_num` passed is always `0`, regardless of any device are available for offload execution. For host-only OpenMP code, this matches the result of `omp_get_num_devices()`, and is a valid result. In the case of devices being available though, this passed identifier is incorrect. While `libomp` calls `omp_get_num_devices()`, `libomptarget` has not fully initialized its PluginManager at that point, hence returning no available devices. Tools relying on `initial_device_num` might therefore incorrectly assume host-side execution when some code runs on a device.
Since the `ompt_get_num_devices()` entry point is also not fully implemented, tools currently need to do on-the-fly handling for the host device.
To make handling easier, consistently use the `omp_initial_device` identifier as the `initial_device_num` and for all device callbacks. This does not rely on knowing the number of available devices during OMPT initialization in the runtime and ensures a consistent value for tools to handle.
Note that the constant `omp_initial_device` is not yet implemented in the OpenMP headers, nor is it accepted by Clang or Flang as a valid device identifier. Therefore, hard-code `-1` as the value, which corresponds the value defined in the OpenMP spec. (OpenMP 6.0, p.534, l.5).
Closes #<!-- -->134451
---
Full diff: https://github.com/llvm/llvm-project/pull/192924.diff
5 Files Affected:
- (modified) offload/libomptarget/OpenMP/OMPT/Callback.cpp (+22-22)
- (modified) offload/libomptarget/device.cpp (+2-2)
- (modified) offload/test/ompt/target_memcpy.c (+1-1)
- (modified) offload/test/ompt/target_memcpy_emi.c (+1-1)
- (modified) openmp/runtime/src/ompt-general.cpp (+3-4)
``````````diff
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 99c8a122c81f4..fa13ebe261da4 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -81,19 +81,19 @@ void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
if (ompt_callback_target_data_op_emi_fn) {
// HostOpId will be set by the tool. Invoke the tool supplied data op EMI
// callback
- ompt_callback_target_data_op_emi_fn(
- ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
- ompt_target_data_alloc, HstPtrBegin,
- /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
- /*TgtDeviceNum=*/DeviceId, Size, Code);
+ ompt_callback_target_data_op_emi_fn(ompt_scope_begin, TargetTaskData,
+ &TargetData, &HostOpId,
+ ompt_target_data_alloc, HstPtrBegin,
+ /*SrcDeviceNum=*/-1, *TgtPtrBegin,
+ /*TgtDeviceNum=*/DeviceId, Size, Code);
} else if (ompt_callback_target_data_op_fn) {
// HostOpId is set by the runtime
HostOpId = createOpId();
// Invoke the tool supplied data op callback
- ompt_callback_target_data_op_fn(
- TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin,
- /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
- /*TgtDeviceNum=*/DeviceId, Size, Code);
+ ompt_callback_target_data_op_fn(TargetData.value, HostOpId,
+ ompt_target_data_alloc, HstPtrBegin,
+ /*SrcDeviceNum=*/-1, *TgtPtrBegin,
+ /*TgtDeviceNum=*/DeviceId, Size, Code);
}
}
@@ -104,11 +104,11 @@ void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
if (ompt_callback_target_data_op_emi_fn) {
// HostOpId will be set by the tool. Invoke the tool supplied data op EMI
// callback
- ompt_callback_target_data_op_emi_fn(
- ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
- ompt_target_data_alloc, HstPtrBegin,
- /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,
- /*TgtDeviceNum=*/DeviceId, Size, Code);
+ ompt_callback_target_data_op_emi_fn(ompt_scope_end, TargetTaskData,
+ &TargetData, &HostOpId,
+ ompt_target_data_alloc, HstPtrBegin,
+ /*SrcDeviceNum=*/-1, *TgtPtrBegin,
+ /*TgtDeviceNum=*/DeviceId, Size, Code);
}
endTargetDataOperation();
}
@@ -333,13 +333,13 @@ void Interface::beginTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin,
if (ompt_callback_target_data_op_emi_fn) {
ompt_callback_target_data_op_emi_fn(
ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
- ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(),
- TgtPtrBegin, DeviceId, Size, Code);
+ ompt_target_data_associate, HstPtrBegin,
+ /*omp_initial_device=*/-1, TgtPtrBegin, DeviceId, Size, Code);
} else if (ompt_callback_target_data_op_fn) {
HostOpId = createOpId();
ompt_callback_target_data_op_fn(
TargetData.value, HostOpId, ompt_target_data_associate, HstPtrBegin,
- omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code);
+ /*omp_initial_device=*/-1, TgtPtrBegin, DeviceId, Size, Code);
}
}
@@ -349,8 +349,8 @@ void Interface::endTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin,
if (ompt_callback_target_data_op_emi_fn) {
ompt_callback_target_data_op_emi_fn(
ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
- ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(),
- TgtPtrBegin, DeviceId, Size, Code);
+ ompt_target_data_associate, HstPtrBegin,
+ /*omp_initial_device=*/-1, TgtPtrBegin, DeviceId, Size, Code);
}
}
@@ -362,13 +362,13 @@ void Interface::beginTargetDisassociatePointer(int64_t DeviceId,
if (ompt_callback_target_data_op_emi_fn) {
ompt_callback_target_data_op_emi_fn(
ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,
- ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(),
+ ompt_target_data_disassociate, HstPtrBegin, /*omp_initial_device=*/-1,
TgtPtrBegin, DeviceId, Size, Code);
} else if (ompt_callback_target_data_op_fn) {
HostOpId = createOpId();
ompt_callback_target_data_op_fn(
TargetData.value, HostOpId, ompt_target_data_disassociate, HstPtrBegin,
- omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code);
+ /*omp_initial_device=*/-1, TgtPtrBegin, DeviceId, Size, Code);
}
}
void Interface::endTargetDisassociatePointer(int64_t DeviceId,
@@ -378,7 +378,7 @@ void Interface::endTargetDisassociatePointer(int64_t DeviceId,
if (ompt_callback_target_data_op_emi_fn) {
ompt_callback_target_data_op_emi_fn(
ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,
- ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(),
+ ompt_target_data_disassociate, HstPtrBegin, /*omp_initial_device=*/-1,
TgtPtrBegin, DeviceId, Size, Code);
}
}
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 12c15aea1ad68..883835f8c0ab2 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -262,7 +262,7 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
OMPT_IF_BUILT(
InterfaceRAII TargetDataSubmitRAII(
RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
- omp_get_initial_device(), HstPtrBegin, DeviceID, TgtPtrBegin, Size,
+ /*omp_initial_device=*/-1, HstPtrBegin, DeviceID, TgtPtrBegin, Size,
/*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
@@ -282,7 +282,7 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
OMPT_IF_BUILT(
InterfaceRAII TargetDataRetrieveRAII(
RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
- DeviceID, TgtPtrBegin, omp_get_initial_device(), HstPtrBegin, Size,
+ DeviceID, TgtPtrBegin, /*omp_initial_device=*/-1, HstPtrBegin, Size,
/*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
diff --git a/offload/test/ompt/target_memcpy.c b/offload/test/ompt/target_memcpy.c
index f769995579f50..fd0c5ed60193d 100644
--- a/offload/test/ompt/target_memcpy.c
+++ b/offload/test/ompt/target_memcpy.c
@@ -49,7 +49,7 @@ int main() {
// clang-format off
/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=ompt_target_data_alloc
-/// CHECK-SAME: src_device_num=[[HOST:[0-9]+]]
+/// CHECK-SAME: src_device_num=[[HOST:-1]]
/// CHECK-SAME: dest_device_num=[[DEVICE:[0-9]+]]
/// CHECK-NOT: code=(nil)
/// CHECK: code=[[CODE1:0x[0-f]+]]
diff --git a/offload/test/ompt/target_memcpy_emi.c b/offload/test/ompt/target_memcpy_emi.c
index 39f262a366f94..c493b08f6f105 100644
--- a/offload/test/ompt/target_memcpy_emi.c
+++ b/offload/test/ompt/target_memcpy_emi.c
@@ -57,7 +57,7 @@ int main(void) {
/// CHECK: Allocating Memory on Device
/// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
-/// CHECK-SAME: src_device_num=[[HOST:[0-9]+]]
+/// CHECK-SAME: src_device_num=[[HOST:-1]]
/// CHECK-SAME: dest_device_num=[[DEVICE:[0-9]+]]
/// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc {{.+}} src_device_num=[[HOST]] {{.+}} dest_device_num=[[DEVICE]]
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 1a778e4ecac3a..d738a06bb5deb 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -468,8 +468,6 @@ void ompt_pre_init() {
#endif
}
-extern "C" int omp_get_initial_device(void);
-
void ompt_post_init() {
//--------------------------------------------------
// Execute the post-initialization logic only once.
@@ -486,7 +484,7 @@ void ompt_post_init() {
//--------------------------------------------------
if (ompt_start_tool_result) {
ompt_enabled.enabled = !!ompt_start_tool_result->initialize(
- ompt_fn_lookup, omp_get_initial_device(),
+ ompt_fn_lookup, /* omp_initial_device */ -1,
&(ompt_start_tool_result->tool_data));
if (!ompt_enabled.enabled) {
@@ -929,7 +927,8 @@ _OMP_EXTERN void ompt_libomp_connect(ompt_start_tool_result_t *result) {
// functions can be extracted and assigned to the callbacks in
// libomptarget
result->initialize(ompt_libomp_target_fn_lookup,
- /* initial_device_num */ 0, /* tool_data */ nullptr);
+ /* initial_device_num=omp_initial_device */ -1,
+ /* tool_data */ nullptr);
// Track the object provided by libomptarget so that the finalizer can be
// called during OMPT finalization
libomptarget_ompt_result = result;
``````````
</details>
https://github.com/llvm/llvm-project/pull/192924
More information about the Openmp-commits
mailing list