[Openmp-commits] [openmp] [OpenMP][NFC] Move mapping related code into OpenMP/Mapping.cpp (PR #75239)
Johannes Doerfert via Openmp-commits
openmp-commits at lists.llvm.org
Tue Dec 12 11:45:49 PST 2023
https://github.com/jdoerfert created https://github.com/llvm/llvm-project/pull/75239
DeviceTy provides an abstraction for "middle-level" operations that can be done with a offload device. Mapping was tied into it but is not strictly necessary. Other languages do not track mapping, and even OpenMP can be used completely without mapping. This simply moves the relevant code into the OpenMP/Mapping.cpp as part of a new class MappingInfoTy. Each device still has one, but it does not clutter the device.cpp anymore.
>From 517a9aad57f9acfe8c18d357b1191d4238b197b1 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes at jdoerfert.de>
Date: Tue, 12 Dec 2023 11:21:46 -0800
Subject: [PATCH] [OpenMP][NFC] Move mapping related code into
OpenMP/Mapping.cpp
DeviceTy provides an abstraction for "middle-level" operations that can
be done with a offload device. Mapping was tied into it but is not
strictly necessary. Other languages do not track mapping, and even
OpenMP can be used completely without mapping. This simply moves the
relevant code into the OpenMP/Mapping.cpp as part of a new class
MappingInfoTy. Each device still has one, but it does not clutter the
device.cpp anymore.
---
openmp/libomptarget/include/ExclusiveAccess.h | 1 +
openmp/libomptarget/include/OpenMP/Mapping.h | 91 ++++
openmp/libomptarget/include/device.h | 96 +---
openmp/libomptarget/src/OpenMP/API.cpp | 19 +-
openmp/libomptarget/src/OpenMP/Mapping.cpp | 488 ++++++++++++++++-
openmp/libomptarget/src/device.cpp | 507 +-----------------
openmp/libomptarget/src/omptarget.cpp | 50 +-
7 files changed, 642 insertions(+), 610 deletions(-)
diff --git a/openmp/libomptarget/include/ExclusiveAccess.h b/openmp/libomptarget/include/ExclusiveAccess.h
index 33f59903637dbc..09b3aac6059dba 100644
--- a/openmp/libomptarget/include/ExclusiveAccess.h
+++ b/openmp/libomptarget/include/ExclusiveAccess.h
@@ -11,6 +11,7 @@
#ifndef OMPTARGET_EXCLUSIVE_ACCESS
#define OMPTARGET_EXCLUSIVE_ACCESS
+#include <cassert>
#include <cstddef>
#include <cstdint>
#include <mutex>
diff --git a/openmp/libomptarget/include/OpenMP/Mapping.h b/openmp/libomptarget/include/OpenMP/Mapping.h
index 9a1ecb80879207..4bd676fc658a7d 100644
--- a/openmp/libomptarget/include/OpenMP/Mapping.h
+++ b/openmp/libomptarget/include/OpenMP/Mapping.h
@@ -13,6 +13,7 @@
#ifndef OMPTARGET_OPENMP_MAPPING_H
#define OMPTARGET_OPENMP_MAPPING_H
+#include "ExclusiveAccess.h"
#include "Shared/EnvironmentVar.h"
#include "omptarget.h"
@@ -443,4 +444,94 @@ int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
void **ArgMappers, AsyncInfoTy &AsyncInfo,
bool FromMapper = false);
+struct MappingInfoTy {
+ MappingInfoTy(DeviceTy &Device) : Device(Device) {}
+
+ /// Host data to device map type with a wrapper key indirection that allows
+ /// concurrent modification of the entries without invalidating the underlying
+ /// entries.
+ using HostDataToTargetListTy =
+ std::set<HostDataToTargetMapKeyTy, std::less<>>;
+
+ /// The HDTTMap is a protected object that can only be accessed by one thread
+ /// at a time.
+ ProtectedObj<HostDataToTargetListTy> HostDataToTargetMap;
+
+ /// The type used to access the HDTT map.
+ using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy;
+
+ /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures
+ /// exclusive access to the HDTT map.
+ LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
+ int64_t Size,
+ HostDataToTargetTy *OwnedTPR = nullptr);
+
+ /// Get the target pointer based on host pointer begin and base. If the
+ /// mapping already exists, the target pointer will be returned directly. In
+ /// addition, if required, the memory region pointed by \p HstPtrBegin of size
+ /// \p Size will also be transferred to the device. If the mapping doesn't
+ /// exist, and if unified shared memory is not enabled, a new mapping will be
+ /// created and the data will also be transferred accordingly. nullptr will be
+ /// returned because of any of following reasons:
+ /// - Data allocation failed;
+ /// - The user tried to do an illegal mapping;
+ /// - Data transfer issue fails.
+ TargetPointerResultTy getTargetPointer(
+ HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
+ int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
+ bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
+ bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+ AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
+ bool ReleaseHDTTMap = true);
+
+ /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
+ /// ensures exclusive access to the HDTT map.
+ void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
+ int64_t Size);
+
+ /// Return the target pointer begin (where the data will be moved).
+ /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
+ /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry
+ /// reference counters will be decremented.
+ /// - \p MustContain enforces that the query must not extend beyond an already
+ /// mapped entry to be valid.
+ /// - \p ForceDelete deletes the entry regardless of its reference counting
+ /// (unless it is infinite).
+ /// - \p FromDataEnd tracks the number of threads referencing the entry at
+ /// targetDataEnd for delayed deletion purpose.
+ [[nodiscard]] TargetPointerResultTy
+ getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
+ bool UseHoldRefCount, bool MustContain = false,
+ bool ForceDelete = false, bool FromDataEnd = false);
+
+ /// Remove the \p Entry from the data map. Expect the entry's total reference
+ /// count to be zero and the caller thread to be the last one using it. \p
+ /// HDTTMap ensure the caller holds exclusive access and can modify the map.
+ /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c
+ /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling
+ /// this function if the map entry is not expected to exist because \p
+ /// HstPtrBegin uses shared memory.
+ [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
+ HostDataToTargetTy *Entry, int64_t Size);
+
+ /// Deallocate the \p Entry from the device memory and delete it. Return \c
+ /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and
+ /// return \c OFFLOAD_FAIL otherwise.
+ [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
+ int64_t Size);
+
+ int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
+ int disassociatePtr(void *HstPtrBegin);
+
+ /// Print information about the transfer from \p HstPtr to \p TgtPtr (or vice
+ /// versa if \p H2D is false). If there is an existing mapping, or if \p Entry
+ /// is set, the associated metadata will be printed as well.
+ void printCopyInfo(void *TgtPtr, void *HstPtr, int64_t Size, bool H2D,
+ HostDataToTargetTy *Entry,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr);
+
+private:
+ DeviceTy &Device;
+};
+
#endif // OMPTARGET_OPENMP_MAPPING_H
diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h
index a84551accaf92a..d28d3c508faf56 100644
--- a/openmp/libomptarget/include/device.h
+++ b/openmp/libomptarget/include/device.h
@@ -53,19 +53,6 @@ struct DeviceTy {
bool HasMappedGlobalData = false;
- /// Host data to device map type with a wrapper key indirection that allows
- /// concurrent modification of the entries without invalidating the underlying
- /// entries.
- using HostDataToTargetListTy =
- std::set<HostDataToTargetMapKeyTy, std::less<>>;
-
- /// The HDTTMap is a protected object that can only be accessed by one thread
- /// at a time.
- ProtectedObj<HostDataToTargetListTy> HostDataToTargetMap;
-
- /// The type used to access the HDTT map.
- using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy;
-
PendingCtorsDtorsPerLibrary PendingCtorsDtors;
std::mutex PendingGlobalsMtx;
@@ -80,71 +67,8 @@ struct DeviceTy {
/// Try to initialize the device and return any failure.
llvm::Error init();
- // Return true if data can be copied to DstDevice directly
- bool isDataExchangable(const DeviceTy &DstDevice);
-
- /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures
- /// exclusive access to the HDTT map.
- LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
- int64_t Size,
- HostDataToTargetTy *OwnedTPR = nullptr);
-
- /// Get the target pointer based on host pointer begin and base. If the
- /// mapping already exists, the target pointer will be returned directly. In
- /// addition, if required, the memory region pointed by \p HstPtrBegin of size
- /// \p Size will also be transferred to the device. If the mapping doesn't
- /// exist, and if unified shared memory is not enabled, a new mapping will be
- /// created and the data will also be transferred accordingly. nullptr will be
- /// returned because of any of following reasons:
- /// - Data allocation failed;
- /// - The user tried to do an illegal mapping;
- /// - Data transfer issue fails.
- TargetPointerResultTy getTargetPointer(
- HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
- int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
- bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
- bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
- AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
- bool ReleaseHDTTMap = true);
-
- /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
- /// ensures exclusive access to the HDTT map.
- void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
- int64_t Size);
-
- /// Return the target pointer begin (where the data will be moved).
- /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
- /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry
- /// reference counters will be decremented.
- /// - \p MustContain enforces that the query must not extend beyond an already
- /// mapped entry to be valid.
- /// - \p ForceDelete deletes the entry regardless of its reference counting
- /// (unless it is infinite).
- /// - \p FromDataEnd tracks the number of threads referencing the entry at
- /// targetDataEnd for delayed deletion purpose.
- [[nodiscard]] TargetPointerResultTy
- getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
- bool UseHoldRefCount, bool MustContain = false,
- bool ForceDelete = false, bool FromDataEnd = false);
-
- /// Remove the \p Entry from the data map. Expect the entry's total reference
- /// count to be zero and the caller thread to be the last one using it. \p
- /// HDTTMap ensure the caller holds exclusive access and can modify the map.
- /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c
- /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling
- /// this function if the map entry is not expected to exist because \p
- /// HstPtrBegin uses shared memory.
- [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
- HostDataToTargetTy *Entry, int64_t Size);
-
- /// Deallocate the \p Entry from the device memory and delete it. Return \c
- /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and
- /// return \c OFFLOAD_FAIL otherwise.
- [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
- int64_t Size);
-
- int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
- int disassociatePtr(void *HstPtrBegin);
+ /// Provide access to the mapping handler.
+ MappingInfoTy &getMappingInfo() { return MappingInfo; }
__tgt_target_table *loadBinary(__tgt_device_image *Img);
@@ -159,6 +83,7 @@ struct DeviceTy {
/// be used (host, shared, device).
void *allocData(int64_t Size, void *HstPtr = nullptr,
int32_t Kind = TARGET_ALLOC_DEFAULT);
+
/// Deallocates memory which \p TgtPtrBegin points at and returns
/// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. p Kind dictates what
/// allocator should be used (host, shared, device).
@@ -170,12 +95,16 @@ struct DeviceTy {
int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
AsyncInfoTy &AsyncInfo,
HostDataToTargetTy *Entry = nullptr,
- DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
// Copy data from device back to host
int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
AsyncInfoTy &AsyncInfo,
HostDataToTargetTy *Entry = nullptr,
- DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
+ // Return true if data can be copied to DstDevice directly
+ bool isDataExchangable(const DeviceTy &DstDevice);
// Copy data from current device to destination device directly
int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
@@ -240,7 +169,12 @@ struct DeviceTy {
void deinit();
/// All offload entries available on this device.
- llvm::DenseMap<llvm::StringRef, OffloadEntryTy *> DeviceOffloadEntries;
+ using DeviceOffloadEntriesMapTy =
+ llvm::DenseMap<llvm::StringRef, OffloadEntryTy *>;
+ ProtectedObj<DeviceOffloadEntriesMapTy> DeviceOffloadEntries;
+
+ /// Handler to collect and organize host-2-device mapping information.
+ MappingInfoTy MappingInfo;
};
#endif
diff --git a/openmp/libomptarget/src/OpenMP/API.cpp b/openmp/libomptarget/src/OpenMP/API.cpp
index b73315f2b77a3f..1769404faf8884 100644
--- a/openmp/libomptarget/src/OpenMP/API.cpp
+++ b/openmp/libomptarget/src/OpenMP/API.cpp
@@ -150,9 +150,9 @@ EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
// only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
// lengh array) is mapped instead of the referred storage.
TargetPointerResultTy TPR =
- DeviceOrErr->getTgtPtrBegin(const_cast<void *>(Ptr), 1,
- /*UpdateRefCount=*/false,
- /*UseHoldRefCount=*/false);
+ DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+ /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
int Rc = TPR.isPresent();
DP("Call to omp_target_is_present returns %d\n", Rc);
return Rc;
@@ -544,8 +544,8 @@ EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset);
- int Rc = DeviceOrErr->associatePtr(const_cast<void *>(HostPtr),
- const_cast<void *>(DeviceAddr), Size);
+ int Rc = DeviceOrErr->getMappingInfo().associatePtr(
+ const_cast<void *>(HostPtr), const_cast<void *>(DeviceAddr), Size);
DP("omp_target_associate_ptr returns %d\n", Rc);
return Rc;
}
@@ -571,7 +571,8 @@ EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
- int Rc = DeviceOrErr->disassociatePtr(const_cast<void *>(HostPtr));
+ int Rc = DeviceOrErr->getMappingInfo().disassociatePtr(
+ const_cast<void *>(HostPtr));
DP("omp_target_disassociate_ptr returns %d\n", Rc);
return Rc;
}
@@ -603,9 +604,9 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
TargetPointerResultTy TPR =
- DeviceOrErr->getTgtPtrBegin(const_cast<void *>(Ptr), 1,
- /*UpdateRefCount=*/false,
- /*UseHoldRefCount=*/false);
+ DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+ /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
if (!TPR.isPresent()) {
DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n",
DPxPTR(Ptr), DeviceNum);
diff --git a/openmp/libomptarget/src/OpenMP/Mapping.cpp b/openmp/libomptarget/src/OpenMP/Mapping.cpp
index c7cb5b6b1e4c45..a5c24810e0af95 100644
--- a/openmp/libomptarget/src/OpenMP/Mapping.cpp
+++ b/openmp/libomptarget/src/OpenMP/Mapping.cpp
@@ -10,13 +10,15 @@
#include "OpenMP/Mapping.h"
+#include "PluginManager.h"
#include "Shared/Debug.h"
+#include "Shared/Requirements.h"
#include "device.h"
/// Dump a table of all the host-target pointer pairs on failure
void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) {
- DeviceTy::HDTTMapAccessorTy HDTTMap =
- Device.HostDataToTargetMap.getExclusiveAccessor();
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
if (HDTTMap->empty())
return;
@@ -38,3 +40,485 @@ void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) {
Info.getLine(), Info.getColumn());
}
}
+
+int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
+ int64_t Size) {
+ HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
+
+ // Check if entry exists
+ auto It = HDTTMap->find(HstPtrBegin);
+ if (It != HDTTMap->end()) {
+ HostDataToTargetTy &HDTT = *It->HDTT;
+ std::lock_guard<HostDataToTargetTy> LG(HDTT);
+ // Mapping already exists
+ bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
+ HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
+ if (IsValid) {
+ DP("Attempt to re-associate the same device ptr+offset with the same "
+ "host ptr, nothing to do\n");
+ return OFFLOAD_SUCCESS;
+ }
+ REPORT("Not allowed to re-associate a different device ptr+offset with "
+ "the same host ptr\n");
+ return OFFLOAD_FAIL;
+ }
+
+ // Mapping does not exist, allocate it with refCount=INF
+ const HostDataToTargetTy &NewEntry =
+ *HDTTMap
+ ->emplace(new HostDataToTargetTy(
+ /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
+ /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
+ /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
+ /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
+ /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
+ /*UseHoldRefCount=*/false, /*Name=*/nullptr,
+ /*IsRefCountINF=*/true))
+ .first->HDTT;
+ DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
+ ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
+ "HoldRefCount=%s\n",
+ DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
+ DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
+ NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
+ (void)NewEntry;
+
+ // Notify the plugin about the new mapping.
+ return Device.notifyDataMapped(HstPtrBegin, Size);
+}
+
+int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
+ HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
+
+ auto It = HDTTMap->find(HstPtrBegin);
+ if (It == HDTTMap->end()) {
+ REPORT("Association not found\n");
+ return OFFLOAD_FAIL;
+ }
+ // Mapping exists
+ HostDataToTargetTy &HDTT = *It->HDTT;
+ std::lock_guard<HostDataToTargetTy> LG(HDTT);
+
+ if (HDTT.getHoldRefCount()) {
+ // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
+ // "It is an error to call acc_unmap_data if the structured reference
+ // count for the pointer is not zero."
+ REPORT("Trying to disassociate a pointer with a non-zero hold reference "
+ "count\n");
+ return OFFLOAD_FAIL;
+ }
+
+ if (HDTT.isDynRefCountInf()) {
+ DP("Association found, removing it\n");
+ void *Event = HDTT.getEvent();
+ delete &HDTT;
+ if (Event)
+ Device.destroyEvent(Event);
+ HDTTMap->erase(It);
+ return Device.notifyDataUnmapped(HstPtrBegin);
+ }
+
+ REPORT("Trying to disassociate a pointer which was not mapped via "
+ "omp_target_associate_ptr\n");
+ return OFFLOAD_FAIL;
+}
+
+LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
+ void *HstPtrBegin, int64_t Size,
+ HostDataToTargetTy *OwnedTPR) {
+
+ uintptr_t HP = (uintptr_t)HstPtrBegin;
+ LookupResult LR;
+
+ DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
+ DPxPTR(HP), Size);
+
+ if (HDTTMap->empty())
+ return LR;
+
+ auto Upper = HDTTMap->upper_bound(HP);
+
+ if (Size == 0) {
+ // specification v5.1 Pointer Initialization for Device Data Environments
+ // upper_bound satisfies
+ // std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
+ if (Upper != HDTTMap->begin()) {
+ LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
+ // the left side of extended address range is satisified.
+ // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
+ // LR.TPR.getEntry()->HstPtrBase
+ LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
+ HP < LR.TPR.getEntry()->HstPtrBase;
+ }
+
+ if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
+ LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
+ // the right side of extended address range is satisified.
+ // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
+ LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
+ }
+ } else {
+ // check the left bin
+ if (Upper != HDTTMap->begin()) {
+ LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
+ // Is it contained?
+ LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
+ HP < LR.TPR.getEntry()->HstPtrEnd &&
+ (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
+ // Does it extend beyond the mapped region?
+ LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
+ (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
+ }
+
+ // check the right bin
+ if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
+ Upper != HDTTMap->end()) {
+ LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
+ // Does it extend into an already mapped region?
+ LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
+ (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
+ // Does it extend beyond the mapped region?
+ LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
+ (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
+ }
+
+ if (LR.Flags.ExtendsBefore) {
+ DP("WARNING: Pointer is not mapped but section extends into already "
+ "mapped data\n");
+ }
+ if (LR.Flags.ExtendsAfter) {
+ DP("WARNING: Pointer is already mapped but section extends beyond mapped "
+ "region\n");
+ }
+ }
+
+ return LR;
+}
+
+TargetPointerResultTy MappingInfoTy::getTargetPointer(
+ HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
+ int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
+ bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
+ bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+ AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
+
+ LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
+ LR.TPR.Flags.IsPresent = true;
+
+ // Release the mapping table lock only after the entry is locked by
+ // attaching it to TPR. Once TPR is destroyed it will release the lock
+ // on entry. If it is returned the lock will move to the returned object.
+ // If LR.Entry is already owned/locked we avoid trying to lock it again.
+
+ // Check if the pointer is contained.
+ // If a variable is mapped to the device manually by the user - which would
+ // lead to the IsContained flag to be true - then we must ensure that the
+ // device address is returned even under unified memory conditions.
+ if (LR.Flags.IsContained ||
+ ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
+ const char *RefCountAction;
+ if (UpdateRefCount) {
+ // After this, reference count >= 1. If the reference count was 0 but the
+ // entry was still there we can reuse the data on the device and avoid a
+ // new submission.
+ LR.TPR.getEntry()->incRefCount(HasHoldModifier);
+ RefCountAction = " (incremented)";
+ } else {
+ // It might have been allocated with the parent, but it's still new.
+ LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
+ RefCountAction = " (update suppressed)";
+ }
+ const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
+ const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
+ uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
+ ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
+ INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
+ "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
+ ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
+ (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
+ Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
+ LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
+ (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
+ LR.TPR.TargetPointer = (void *)Ptr;
+ } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
+ // Explicit extension of mapped data - not allowed.
+ MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
+ " (%" PRId64
+ " bytes), but device allocation maps to host at " DPxMOD
+ " (%" PRId64 " bytes)",
+ DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
+ LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
+ if (HasPresentModifier)
+ MESSAGE("device mapping required by 'present' map type modifier does not "
+ "exist for host address " DPxMOD " (%" PRId64 " bytes)",
+ DPxPTR(HstPtrBegin), Size);
+ } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
+ !HasCloseModifier) {
+ // If unified shared memory is active, implicitly mapped variables that are
+ // not privatized use host address. Any explicitly mapped variables also use
+ // host address where correctness is not impeded. In all other cases maps
+ // are respected.
+ // In addition to the mapping rules above, the close map modifier forces the
+ // mapping of the variable to the device.
+ if (Size) {
+ DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
+ "memory\n",
+ DPxPTR((uintptr_t)HstPtrBegin), Size);
+ LR.TPR.Flags.IsPresent = false;
+ LR.TPR.Flags.IsHostPointer = true;
+ LR.TPR.TargetPointer = HstPtrBegin;
+ }
+ } else if (HasPresentModifier) {
+ DP("Mapping required by 'present' map type modifier does not exist for "
+ "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
+ DPxPTR(HstPtrBegin), Size);
+ MESSAGE("device mapping required by 'present' map type modifier does not "
+ "exist for host address " DPxMOD " (%" PRId64 " bytes)",
+ DPxPTR(HstPtrBegin), Size);
+ } else if (Size) {
+ // If it is not contained and Size > 0, we should create a new entry for it.
+ LR.TPR.Flags.IsNewEntry = true;
+ uintptr_t TgtAllocBegin =
+ (uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
+ uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
+ // Release the mapping table lock only after the entry is locked by
+ // attaching it to TPR.
+ LR.TPR.setEntry(HDTTMap
+ ->emplace(new HostDataToTargetTy(
+ (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
+ (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
+ TgtPtrBegin, HasHoldModifier, HstPtrName))
+ .first->HDTT);
+ INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
+ "Creating new map entry with HstPtrBase=" DPxMOD
+ ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
+ ", TgtPtrBegin=" DPxMOD
+ ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
+ DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
+ DPxPTR(TgtPtrBegin), Size,
+ LR.TPR.getEntry()->dynRefCountToStr().c_str(),
+ LR.TPR.getEntry()->holdRefCountToStr().c_str(),
+ (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
+ LR.TPR.TargetPointer = (void *)TgtPtrBegin;
+
+ // Notify the plugin about the new mapping.
+ if (Device.notifyDataMapped(HstPtrBegin, Size))
+ return {{false /* IsNewEntry */, false /* IsHostPointer */},
+ nullptr /* Entry */,
+ nullptr /* TargetPointer */};
+ } else {
+ // This entry is not present and we did not create a new entry for it.
+ LR.TPR.Flags.IsPresent = false;
+ }
+
+ // All mapping table modifications have been made. If the user requested it we
+ // give up the lock.
+ if (ReleaseHDTTMap)
+ HDTTMap.destroy();
+
+ // If the target pointer is valid, and we need to transfer data, issue the
+ // data transfer.
+ if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
+ (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
+ DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
+ DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
+
+ int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
+ AsyncInfo, LR.TPR.getEntry());
+ if (Ret != OFFLOAD_SUCCESS) {
+ REPORT("Copying data to device failed.\n");
+ // We will also return nullptr if the data movement fails because that
+ // pointer points to a corrupted memory region so it doesn't make any
+ // sense to continue to use it.
+ LR.TPR.TargetPointer = nullptr;
+ } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
+ OFFLOAD_SUCCESS)
+ return {{false /* IsNewEntry */, false /* IsHostPointer */},
+ nullptr /* Entry */,
+ nullptr /* TargetPointer */};
+ } else {
+ // If not a host pointer and no present modifier, we need to wait for the
+ // event if it exists.
+ // Note: Entry might be nullptr because of zero length array section.
+ if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
+ !HasPresentModifier) {
+ void *Event = LR.TPR.getEntry()->getEvent();
+ if (Event) {
+ int Ret = Device.waitEvent(Event, AsyncInfo);
+ if (Ret != OFFLOAD_SUCCESS) {
+ // If it fails to wait for the event, we need to return nullptr in
+ // case of any data race.
+ REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
+ return {{false /* IsNewEntry */, false /* IsHostPointer */},
+ nullptr /* Entry */,
+ nullptr /* TargetPointer */};
+ }
+ }
+ }
+ }
+
+ return std::move(LR.TPR);
+}
+
+TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
+ void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
+ bool MustContain, bool ForceDelete, bool FromDataEnd) {
+ HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
+
+ LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
+
+ LR.TPR.Flags.IsPresent = true;
+
+ if (LR.Flags.IsContained ||
+ (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
+ LR.TPR.Flags.IsLast =
+ LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
+
+ if (ForceDelete) {
+ LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
+ assert(LR.TPR.Flags.IsLast ==
+ LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
+ "expected correct IsLast prediction for reset");
+ }
+
+ // Increment the number of threads that is using the entry on a
+ // targetDataEnd, tracking the number of possible "deleters". A thread may
+ // come to own the entry deletion even if it was not the last one querying
+ // for it. Thus, we must track every query on targetDataEnds to ensure only
+ // the last thread that holds a reference to an entry actually deletes it.
+ if (FromDataEnd)
+ LR.TPR.getEntry()->incDataEndThreadCount();
+
+ const char *RefCountAction;
+ if (!UpdateRefCount) {
+ RefCountAction = " (update suppressed)";
+ } else if (LR.TPR.Flags.IsLast) {
+ LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
+ assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
+ "Expected zero reference count when deletion is scheduled");
+ if (ForceDelete)
+ RefCountAction = " (reset, delayed deletion)";
+ else
+ RefCountAction = " (decremented, delayed deletion)";
+ } else {
+ LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
+ RefCountAction = " (decremented)";
+ }
+ const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
+ const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
+ uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
+ ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
+ INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
+ "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
+ "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
+ DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
+ LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
+ LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
+ LR.TPR.TargetPointer = (void *)TP;
+ } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) {
+ // If the value isn't found in the mapping and unified shared memory
+ // is on then it means we have stumbled upon a value which we need to
+ // use directly from the host.
+ DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
+ "memory\n",
+ DPxPTR((uintptr_t)HstPtrBegin), Size);
+ LR.TPR.Flags.IsPresent = false;
+ LR.TPR.Flags.IsHostPointer = true;
+ LR.TPR.TargetPointer = HstPtrBegin;
+ } else {
+ // OpenMP Specification v5.2: if a matching list item is not found, the
+ // pointer retains its original value as per firstprivate semantics.
+ LR.TPR.Flags.IsPresent = false;
+ LR.TPR.Flags.IsHostPointer = false;
+ LR.TPR.TargetPointer = HstPtrBegin;
+ }
+
+ return std::move(LR.TPR);
+}
+
+// Return the target pointer begin (where the data will be moved).
+void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap,
+ void *HstPtrBegin, int64_t Size) {
+ uintptr_t HP = (uintptr_t)HstPtrBegin;
+ LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
+ if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
+ uintptr_t TP =
+ LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
+ return (void *)TP;
+ }
+
+ return NULL;
+}
+
+int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
+ HostDataToTargetTy *Entry, int64_t Size) {
+ assert(Entry && "Trying to delete a null entry from the HDTT map.");
+ assert(Entry->getTotalRefCount() == 0 &&
+ Entry->getDataEndThreadCount() == 0 &&
+ "Trying to delete entry that is in use or owned by another thread.");
+
+ INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
+ "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
+ ", Size=%" PRId64 ", Name=%s\n",
+ DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
+ (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
+ : "unknown");
+
+ if (HDTTMap->erase(Entry) == 0) {
+ REPORT("Trying to remove a non-existent map entry\n");
+ return OFFLOAD_FAIL;
+ }
+
+ return OFFLOAD_SUCCESS;
+}
+
+int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
+ int64_t Size) {
+ assert(Entry && "Trying to deallocate a null entry.");
+
+ DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
+ "starting at " DPxMOD "\n",
+ DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
+
+ void *Event = Entry->getEvent();
+ if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) {
+ REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
+ return OFFLOAD_FAIL;
+ }
+
+ int Ret = Device.deleteData((void *)Entry->TgtAllocBegin);
+
+ // Notify the plugin about the unmapped memory.
+ Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin);
+
+ delete Entry;
+
+ return Ret;
+}
+
+static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
+ void *DstPtrBegin, int64_t Size,
+ HostDataToTargetTy *HT) {
+
+ INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
+ "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
+ ", Size=%" PRId64 ", Name=%s\n",
+ H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
+ DPxPTR(SrcPtrBegin), H2D ? "Tgt" : "Hst", DPxPTR(DstPtrBegin), Size,
+ (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
+ : "unknown");
+}
+
+void MappingInfoTy::printCopyInfo(
+ void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D,
+ HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+ auto HDTTMap =
+ HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
+ LookupResult LR;
+ if (!Entry) {
+ LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
+ Entry = LR.TPR.getEntry();
+ }
+ printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size,
+ Entry);
+}
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index 01fc3232887612..dbad13b92bcc14 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -12,6 +12,7 @@
#include "device.h"
#include "OffloadEntry.h"
+#include "OpenMP/Mapping.h"
#include "OpenMP/OMPT/Callback.h"
#include "OpenMP/OMPT/Interface.h"
#include "PluginManager.h"
@@ -65,7 +66,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
: DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
- PendingCtorsDtors(), PendingGlobalsMtx() {}
+ PendingCtorsDtors(), PendingGlobalsMtx(), MappingInfo(*this) {}
DeviceTy::~DeviceTy() {
if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE))
@@ -75,460 +76,6 @@ DeviceTy::~DeviceTy() {
dumpTargetPointerMappings(&Loc, *this);
}
-int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
- HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
-
- // Check if entry exists
- auto It = HDTTMap->find(HstPtrBegin);
- if (It != HDTTMap->end()) {
- HostDataToTargetTy &HDTT = *It->HDTT;
- std::lock_guard<HostDataToTargetTy> LG(HDTT);
- // Mapping already exists
- bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
- HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
- if (IsValid) {
- DP("Attempt to re-associate the same device ptr+offset with the same "
- "host ptr, nothing to do\n");
- return OFFLOAD_SUCCESS;
- }
- REPORT("Not allowed to re-associate a different device ptr+offset with "
- "the same host ptr\n");
- return OFFLOAD_FAIL;
- }
-
- // Mapping does not exist, allocate it with refCount=INF
- const HostDataToTargetTy &NewEntry =
- *HDTTMap
- ->emplace(new HostDataToTargetTy(
- /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
- /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
- /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
- /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
- /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
- /*UseHoldRefCount=*/false, /*Name=*/nullptr,
- /*IsRefCountINF=*/true))
- .first->HDTT;
- DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
- ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
- "HoldRefCount=%s\n",
- DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
- DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
- NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
- (void)NewEntry;
-
- // Notify the plugin about the new mapping.
- return notifyDataMapped(HstPtrBegin, Size);
-}
-
-int DeviceTy::disassociatePtr(void *HstPtrBegin) {
- HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
-
- auto It = HDTTMap->find(HstPtrBegin);
- if (It == HDTTMap->end()) {
- REPORT("Association not found\n");
- return OFFLOAD_FAIL;
- }
- // Mapping exists
- HostDataToTargetTy &HDTT = *It->HDTT;
- std::lock_guard<HostDataToTargetTy> LG(HDTT);
-
- if (HDTT.getHoldRefCount()) {
- // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
- // "It is an error to call acc_unmap_data if the structured reference
- // count for the pointer is not zero."
- REPORT("Trying to disassociate a pointer with a non-zero hold reference "
- "count\n");
- return OFFLOAD_FAIL;
- }
-
- if (HDTT.isDynRefCountInf()) {
- DP("Association found, removing it\n");
- void *Event = HDTT.getEvent();
- delete &HDTT;
- if (Event)
- destroyEvent(Event);
- HDTTMap->erase(It);
- return notifyDataUnmapped(HstPtrBegin);
- }
-
- REPORT("Trying to disassociate a pointer which was not mapped via "
- "omp_target_associate_ptr\n");
- return OFFLOAD_FAIL;
-}
-
-LookupResult DeviceTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
- void *HstPtrBegin, int64_t Size,
- HostDataToTargetTy *OwnedTPR) {
-
- uintptr_t HP = (uintptr_t)HstPtrBegin;
- LookupResult LR;
-
- DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
- DPxPTR(HP), Size);
-
- if (HDTTMap->empty())
- return LR;
-
- auto Upper = HDTTMap->upper_bound(HP);
-
- if (Size == 0) {
- // specification v5.1 Pointer Initialization for Device Data Environments
- // upper_bound satisfies
- // std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
- if (Upper != HDTTMap->begin()) {
- LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
- // the left side of extended address range is satisified.
- // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
- // LR.TPR.getEntry()->HstPtrBase
- LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
- HP < LR.TPR.getEntry()->HstPtrBase;
- }
-
- if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
- LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
- // the right side of extended address range is satisified.
- // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
- LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
- }
- } else {
- // check the left bin
- if (Upper != HDTTMap->begin()) {
- LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
- // Is it contained?
- LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
- HP < LR.TPR.getEntry()->HstPtrEnd &&
- (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
- // Does it extend beyond the mapped region?
- LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
- (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
- }
-
- // check the right bin
- if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
- Upper != HDTTMap->end()) {
- LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
- // Does it extend into an already mapped region?
- LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
- (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
- // Does it extend beyond the mapped region?
- LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
- (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
- }
-
- if (LR.Flags.ExtendsBefore) {
- DP("WARNING: Pointer is not mapped but section extends into already "
- "mapped data\n");
- }
- if (LR.Flags.ExtendsAfter) {
- DP("WARNING: Pointer is already mapped but section extends beyond mapped "
- "region\n");
- }
- }
-
- return LR;
-}
-
-TargetPointerResultTy DeviceTy::getTargetPointer(
- HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
- int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
- bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
- bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
- AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
-
- LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
- LR.TPR.Flags.IsPresent = true;
-
- // Release the mapping table lock only after the entry is locked by
- // attaching it to TPR. Once TPR is destroyed it will release the lock
- // on entry. If it is returned the lock will move to the returned object.
- // If LR.Entry is already owned/locked we avoid trying to lock it again.
-
- // Check if the pointer is contained.
- // If a variable is mapped to the device manually by the user - which would
- // lead to the IsContained flag to be true - then we must ensure that the
- // device address is returned even under unified memory conditions.
- if (LR.Flags.IsContained ||
- ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
- const char *RefCountAction;
- if (UpdateRefCount) {
- // After this, reference count >= 1. If the reference count was 0 but the
- // entry was still there we can reuse the data on the device and avoid a
- // new submission.
- LR.TPR.getEntry()->incRefCount(HasHoldModifier);
- RefCountAction = " (incremented)";
- } else {
- // It might have been allocated with the parent, but it's still new.
- LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
- RefCountAction = " (update suppressed)";
- }
- const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
- const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
- uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
- ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
- INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID,
- "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
- ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
- (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
- Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
- LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
- (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
- LR.TPR.TargetPointer = (void *)Ptr;
- } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
- // Explicit extension of mapped data - not allowed.
- MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
- " (%" PRId64
- " bytes), but device allocation maps to host at " DPxMOD
- " (%" PRId64 " bytes)",
- DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
- LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
- if (HasPresentModifier)
- MESSAGE("device mapping required by 'present' map type modifier does not "
- "exist for host address " DPxMOD " (%" PRId64 " bytes)",
- DPxPTR(HstPtrBegin), Size);
- } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
- !HasCloseModifier) {
- // If unified shared memory is active, implicitly mapped variables that are
- // not privatized use host address. Any explicitly mapped variables also use
- // host address where correctness is not impeded. In all other cases maps
- // are respected.
- // In addition to the mapping rules above, the close map modifier forces the
- // mapping of the variable to the device.
- if (Size) {
- DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
- "memory\n",
- DPxPTR((uintptr_t)HstPtrBegin), Size);
- LR.TPR.Flags.IsPresent = false;
- LR.TPR.Flags.IsHostPointer = true;
- LR.TPR.TargetPointer = HstPtrBegin;
- }
- } else if (HasPresentModifier) {
- DP("Mapping required by 'present' map type modifier does not exist for "
- "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
- DPxPTR(HstPtrBegin), Size);
- MESSAGE("device mapping required by 'present' map type modifier does not "
- "exist for host address " DPxMOD " (%" PRId64 " bytes)",
- DPxPTR(HstPtrBegin), Size);
- } else if (Size) {
- // If it is not contained and Size > 0, we should create a new entry for it.
- LR.TPR.Flags.IsNewEntry = true;
- uintptr_t TgtAllocBegin =
- (uintptr_t)allocData(TgtPadding + Size, HstPtrBegin);
- uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
- // Release the mapping table lock only after the entry is locked by
- // attaching it to TPR.
- LR.TPR.setEntry(HDTTMap
- ->emplace(new HostDataToTargetTy(
- (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
- (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
- TgtPtrBegin, HasHoldModifier, HstPtrName))
- .first->HDTT);
- INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID,
- "Creating new map entry with HstPtrBase=" DPxMOD
- ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
- ", TgtPtrBegin=" DPxMOD
- ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
- DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
- DPxPTR(TgtPtrBegin), Size,
- LR.TPR.getEntry()->dynRefCountToStr().c_str(),
- LR.TPR.getEntry()->holdRefCountToStr().c_str(),
- (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
- LR.TPR.TargetPointer = (void *)TgtPtrBegin;
-
- // Notify the plugin about the new mapping.
- if (notifyDataMapped(HstPtrBegin, Size))
- return {{false /* IsNewEntry */, false /* IsHostPointer */},
- nullptr /* Entry */,
- nullptr /* TargetPointer */};
- } else {
- // This entry is not present and we did not create a new entry for it.
- LR.TPR.Flags.IsPresent = false;
- }
-
- // All mapping table modifications have been made. If the user requested it we
- // give up the lock.
- if (ReleaseHDTTMap)
- HDTTMap.destroy();
-
- // If the target pointer is valid, and we need to transfer data, issue the
- // data transfer.
- if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
- (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
- DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
- DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
-
- int Ret = submitData(LR.TPR.TargetPointer, HstPtrBegin, Size, AsyncInfo,
- LR.TPR.getEntry());
- if (Ret != OFFLOAD_SUCCESS) {
- REPORT("Copying data to device failed.\n");
- // We will also return nullptr if the data movement fails because that
- // pointer points to a corrupted memory region so it doesn't make any
- // sense to continue to use it.
- LR.TPR.TargetPointer = nullptr;
- } else if (LR.TPR.getEntry()->addEventIfNecessary(*this, AsyncInfo) !=
- OFFLOAD_SUCCESS)
- return {{false /* IsNewEntry */, false /* IsHostPointer */},
- nullptr /* Entry */,
- nullptr /* TargetPointer */};
- } else {
- // If not a host pointer and no present modifier, we need to wait for the
- // event if it exists.
- // Note: Entry might be nullptr because of zero length array section.
- if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
- !HasPresentModifier) {
- void *Event = LR.TPR.getEntry()->getEvent();
- if (Event) {
- int Ret = waitEvent(Event, AsyncInfo);
- if (Ret != OFFLOAD_SUCCESS) {
- // If it fails to wait for the event, we need to return nullptr in
- // case of any data race.
- REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
- return {{false /* IsNewEntry */, false /* IsHostPointer */},
- nullptr /* Entry */,
- nullptr /* TargetPointer */};
- }
- }
- }
- }
-
- return std::move(LR.TPR);
-}
-
-TargetPointerResultTy
-DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
- bool UseHoldRefCount, bool MustContain,
- bool ForceDelete, bool FromDataEnd) {
- HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
-
- LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
-
- LR.TPR.Flags.IsPresent = true;
-
- if (LR.Flags.IsContained ||
- (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
- LR.TPR.Flags.IsLast =
- LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
-
- if (ForceDelete) {
- LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
- assert(LR.TPR.Flags.IsLast ==
- LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
- "expected correct IsLast prediction for reset");
- }
-
- // Increment the number of threads that is using the entry on a
- // targetDataEnd, tracking the number of possible "deleters". A thread may
- // come to own the entry deletion even if it was not the last one querying
- // for it. Thus, we must track every query on targetDataEnds to ensure only
- // the last thread that holds a reference to an entry actually deletes it.
- if (FromDataEnd)
- LR.TPR.getEntry()->incDataEndThreadCount();
-
- const char *RefCountAction;
- if (!UpdateRefCount) {
- RefCountAction = " (update suppressed)";
- } else if (LR.TPR.Flags.IsLast) {
- LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
- assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
- "Expected zero reference count when deletion is scheduled");
- if (ForceDelete)
- RefCountAction = " (reset, delayed deletion)";
- else
- RefCountAction = " (decremented, delayed deletion)";
- } else {
- LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
- RefCountAction = " (decremented)";
- }
- const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
- const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
- uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
- ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
- INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID,
- "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
- "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
- DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
- LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
- LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
- LR.TPR.TargetPointer = (void *)TP;
- } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) {
- // If the value isn't found in the mapping and unified shared memory
- // is on then it means we have stumbled upon a value which we need to
- // use directly from the host.
- DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
- "memory\n",
- DPxPTR((uintptr_t)HstPtrBegin), Size);
- LR.TPR.Flags.IsPresent = false;
- LR.TPR.Flags.IsHostPointer = true;
- LR.TPR.TargetPointer = HstPtrBegin;
- } else {
- // OpenMP Specification v5.2: if a matching list item is not found, the
- // pointer retains its original value as per firstprivate semantics.
- LR.TPR.Flags.IsPresent = false;
- LR.TPR.Flags.IsHostPointer = false;
- LR.TPR.TargetPointer = HstPtrBegin;
- }
-
- return std::move(LR.TPR);
-}
-
-// Return the target pointer begin (where the data will be moved).
-void *DeviceTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
- int64_t Size) {
- uintptr_t HP = (uintptr_t)HstPtrBegin;
- LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
- if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
- uintptr_t TP =
- LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
- return (void *)TP;
- }
-
- return NULL;
-}
-
-int DeviceTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
- HostDataToTargetTy *Entry, int64_t Size) {
- assert(Entry && "Trying to delete a null entry from the HDTT map.");
- assert(Entry->getTotalRefCount() == 0 &&
- Entry->getDataEndThreadCount() == 0 &&
- "Trying to delete entry that is in use or owned by another thread.");
-
- INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID,
- "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
- ", Size=%" PRId64 ", Name=%s\n",
- DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
- (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
- : "unknown");
-
- if (HDTTMap->erase(Entry) == 0) {
- REPORT("Trying to remove a non-existent map entry\n");
- return OFFLOAD_FAIL;
- }
-
- return OFFLOAD_SUCCESS;
-}
-
-int DeviceTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry, int64_t Size) {
- assert(Entry && "Trying to deallocate a null entry.");
-
- DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
- "starting at " DPxMOD "\n",
- DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
-
- void *Event = Entry->getEvent();
- if (Event && destroyEvent(Event) != OFFLOAD_SUCCESS) {
- REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
- return OFFLOAD_FAIL;
- }
-
- int Ret = deleteData((void *)Entry->TgtAllocBegin);
-
- // Notify the plugin about the unmapped memory.
- Ret |= notifyDataUnmapped((void *)Entry->HstPtrBegin);
-
- delete Entry;
-
- return Ret;
-}
-
llvm::Error DeviceTy::init() {
// Make call to init_requires if it exists for this plugin.
int32_t Ret = 0;
@@ -586,34 +133,13 @@ int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
}
-static void printCopyInfo(int DeviceId, bool H2D, void *SrcPtrBegin,
- void *DstPtrBegin, int64_t Size,
- HostDataToTargetTy *HT) {
-
- INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
- "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
- ", Size=%" PRId64 ", Name=%s\n",
- H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
- DPxPTR(SrcPtrBegin), H2D ? "Tgt" : "Hst", DPxPTR(DstPtrBegin), Size,
- (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
- : "unknown");
-}
-
// Submit data to device
int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry,
- DeviceTy::HDTTMapAccessorTy *HDTTMapPtr) {
- if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
- HDTTMapAccessorTy HDTTMap =
- HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
- LookupResult LR;
- if (!Entry) {
- LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
- Entry = LR.TPR.getEntry();
- }
- printCopyInfo(DeviceID, /* H2D */ true, HstPtrBegin, TgtPtrBegin, Size,
- Entry);
- }
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+ if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
+ MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/true,
+ Entry, HDTTMapPtr);
/// RAII to establish tool anchors before and after data submit
OMPT_IF_BUILT(
@@ -632,18 +158,10 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
int64_t Size, AsyncInfoTy &AsyncInfo,
HostDataToTargetTy *Entry,
- DeviceTy::HDTTMapAccessorTy *HDTTMapPtr) {
- if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
- HDTTMapAccessorTy HDTTMap =
- HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
- LookupResult LR;
- if (!Entry) {
- LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
- Entry = LR.TPR.getEntry();
- }
- printCopyInfo(DeviceID, /* H2D */ false, TgtPtrBegin, HstPtrBegin, Size,
- Entry);
- }
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+ if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
+ MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/false,
+ Entry, HDTTMapPtr);
/// RAII to establish tool anchors before and after data retrieval
OMPT_IF_BUILT(
@@ -775,7 +293,8 @@ int32_t DeviceTy::destroyEvent(void *Event) {
void DeviceTy::addOffloadEntry(OffloadEntryTy &Entry) {
std::lock_guard<decltype(PendingGlobalsMtx)> Lock(PendingGlobalsMtx);
- DeviceOffloadEntries[Entry.getName()] = &Entry;
+ DeviceOffloadEntries.getExclusiveAccessor()->insert(
+ {Entry.getName(), &Entry});
if (Entry.isGlobal())
return;
@@ -808,7 +327,7 @@ void DeviceTy::addOffloadEntry(OffloadEntryTy &Entry) {
void DeviceTy::dumpOffloadEntries() {
fprintf(stderr, "Device %i offload entries:\n", DeviceID);
- for (auto &It : DeviceOffloadEntries) {
+ for (auto &It : *DeviceOffloadEntries.getExclusiveAccessor()) {
const char *Kind = "kernel";
if (It.second->isCTor())
Kind = "constructor";
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 2edbadaa6e02c6..0d16a41c7616c7 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -194,8 +194,8 @@ static int initLibrary(DeviceTy &Device) {
break;
}
- DeviceTy::HDTTMapAccessorTy HDTTMap =
- Device.HostDataToTargetMap.getExclusiveAccessor();
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
__tgt_target_table *HostTable = &TransTable->HostTable;
for (__tgt_offload_entry *CurrDeviceEntry = TargetTable->EntriesBegin,
@@ -213,8 +213,8 @@ static int initLibrary(DeviceTy &Device) {
// therefore we must allow for multiple weak symbols to be loaded from
// the fat binary. Treat these mappings as any other "regular"
// mapping. Add entry to map.
- if (Device.getTgtPtrBegin(HDTTMap, CurrHostEntry->addr,
- CurrHostEntry->size))
+ if (Device.getMappingInfo().getTgtPtrBegin(HDTTMap, CurrHostEntry->addr,
+ CurrHostEntry->size))
continue;
void *CurrDeviceEntryAddr = CurrDeviceEntry->addr;
@@ -604,8 +604,8 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
bool UpdateRef =
!(ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) && !(FromMapper && I == 0);
- DeviceTy::HDTTMapAccessorTy HDTTMap =
- Device.HostDataToTargetMap.getExclusiveAccessor();
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
DP("Has a pointer entry: \n");
// Base is address of pointer.
@@ -621,7 +621,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
// entry for a global that might not already be allocated by the time the
// PTR_AND_OBJ entry is handled below, and so the allocation might fail
// when HasPresentModifier.
- PointerTpr = Device.getTargetPointer(
+ PointerTpr = Device.getMappingInfo().getTargetPointer(
HDTTMap, HstPtrBase, HstPtrBase, /*TgtPadding=*/0, sizeof(void *),
/*HstPtrName=*/nullptr,
/*HasFlagTo=*/false, /*HasFlagAlways=*/false, IsImplicit, UpdateRef,
@@ -651,7 +651,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
const bool HasFlagTo = ArgTypes[I] & OMP_TGT_MAPTYPE_TO;
const bool HasFlagAlways = ArgTypes[I] & OMP_TGT_MAPTYPE_ALWAYS;
// Note that HDTTMap will be released in getTargetPointer.
- auto TPR = Device.getTargetPointer(
+ auto TPR = Device.getMappingInfo().getTargetPointer(
HDTTMap, HstPtrBegin, HstPtrBase, TgtPadding, DataSize, HstPtrName,
HasFlagTo, HasFlagAlways, IsImplicit, UpdateRef, HasCloseModifier,
HasPresentModifier, HasHoldModifier, AsyncInfo, PointerTpr.getEntry());
@@ -768,8 +768,8 @@ postProcessingTargetDataEnd(DeviceTy *Device,
// will avoid another thread reusing the entry now. Note that we do
// not request (exclusive) access to the HDTT map if DelEntry is
// not set.
- DeviceTy::HDTTMapAccessorTy HDTTMap =
- Device->HostDataToTargetMap.getExclusiveAccessor();
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device->getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
// We cannot use a lock guard because we may end up delete the mutex.
// We also explicitly unlocked the entry after it was put in the EntriesInfo
@@ -807,10 +807,10 @@ postProcessingTargetDataEnd(DeviceTy *Device,
if (!DelEntry)
continue;
- Ret = Device->eraseMapEntry(HDTTMap, Entry, DataSize);
+ Ret = Device->getMappingInfo().eraseMapEntry(HDTTMap, Entry, DataSize);
// Entry is already remove from the map, we can unlock it now.
HDTTMap.destroy();
- Ret |= Device->deallocTgtPtrAndEntry(Entry, DataSize);
+ Ret |= Device->getMappingInfo().deallocTgtPtrAndEntry(Entry, DataSize);
if (Ret != OFFLOAD_SUCCESS) {
REPORT("Deallocating data from device failed.\n");
break;
@@ -868,9 +868,9 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
bool HasHoldModifier = ArgTypes[I] & OMP_TGT_MAPTYPE_OMPX_HOLD;
// If PTR_AND_OBJ, HstPtrBegin is address of pointee
- TargetPointerResultTy TPR =
- Device.getTgtPtrBegin(HstPtrBegin, DataSize, UpdateRef, HasHoldModifier,
- !IsImplicit, ForceDelete, /*FromDataEnd=*/true);
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ HstPtrBegin, DataSize, UpdateRef, HasHoldModifier, !IsImplicit,
+ ForceDelete, /*FromDataEnd=*/true);
void *TgtPtrBegin = TPR.TargetPointer;
if (!TPR.isPresent() && !TPR.isHostPointer() &&
(DataSize || HasPresentModifier)) {
@@ -965,9 +965,9 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
static int targetDataContiguous(ident_t *Loc, DeviceTy &Device, void *ArgsBase,
void *HstPtrBegin, int64_t ArgSize,
int64_t ArgType, AsyncInfoTy &AsyncInfo) {
- TargetPointerResultTy TPR =
- Device.getTgtPtrBegin(HstPtrBegin, ArgSize, /*UpdateRefCount=*/false,
- /*UseHoldRefCount=*/false, /*MustContain=*/true);
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ HstPtrBegin, ArgSize, /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false, /*MustContain=*/true);
void *TgtPtrBegin = TPR.TargetPointer;
if (!TPR.isPresent()) {
DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin));
@@ -1445,9 +1445,10 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr,
uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase;
void *TgtPtrBegin = (void *)((uintptr_t)TgtPtrBase + Delta);
void *&PointerTgtPtrBegin = AsyncInfo.getVoidPtrLocation();
- TargetPointerResultTy TPR = DeviceOrErr->getTgtPtrBegin(
- HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false,
- /*UseHoldRefCount=*/false);
+ TargetPointerResultTy TPR =
+ DeviceOrErr->getMappingInfo().getTgtPtrBegin(
+ HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
PointerTgtPtrBegin = TPR.TargetPointer;
if (!TPR.isPresent()) {
DP("No lambda captured variable mapped (" DPxMOD ") - ignored\n",
@@ -1503,9 +1504,10 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr,
} else {
if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)
HstPtrBase = *reinterpret_cast<void **>(HstPtrBase);
- TPR = DeviceOrErr->getTgtPtrBegin(HstPtrBegin, ArgSizes[I],
- /*UpdateRefCount=*/false,
- /*UseHoldRefCount=*/false);
+ TPR = DeviceOrErr->getMappingInfo().getTgtPtrBegin(
+ HstPtrBegin, ArgSizes[I],
+ /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
TgtPtrBegin = TPR.TargetPointer;
TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
#ifdef OMPTARGET_DEBUG
More information about the Openmp-commits
mailing list