[flang-commits] [llvm] [libcxx] [clang] [openmp] [flang] [OpenMP][NFC] Separate OpenMP/OpenACC specific mapping code (PR #73817)
Johannes Doerfert via flang-commits
flang-commits at lists.llvm.org
Wed Nov 29 10:28:55 PST 2023
https://github.com/jdoerfert updated https://github.com/llvm/llvm-project/pull/73817
>From ed1513641d575c4a2881613864c892aff7855a78 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes at jdoerfert.de>
Date: Tue, 28 Nov 2023 18:51:23 -0800
Subject: [PATCH] [OpenMP][NFC] Separate OpenMP/OpenACC specific mapping code
While this does not really encapsulate the mapping code, it at least
moves most of the declarations out of the way.
---
openmp/libomptarget/include/OpenMP/Mapping.h | 427 +++++++++++++++++++
openmp/libomptarget/include/device.h | 350 +--------------
openmp/libomptarget/src/CMakeLists.txt | 2 +
openmp/libomptarget/src/OpenMP/Mapping.cpp | 40 ++
openmp/libomptarget/src/omptarget.cpp | 1 +
openmp/libomptarget/src/private.h | 82 ----
6 files changed, 473 insertions(+), 429 deletions(-)
create mode 100644 openmp/libomptarget/include/OpenMP/Mapping.h
create mode 100644 openmp/libomptarget/src/OpenMP/Mapping.cpp
diff --git a/openmp/libomptarget/include/OpenMP/Mapping.h b/openmp/libomptarget/include/OpenMP/Mapping.h
new file mode 100644
index 000000000000000..b01831c61f6c823
--- /dev/null
+++ b/openmp/libomptarget/include/OpenMP/Mapping.h
@@ -0,0 +1,427 @@
+//===-- OpenMP/Mapping.h - OpenMP/OpenACC pointer mapping -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Declarations for managing host-to-device pointer mappings.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_OPENMP_MAPPING_H
+#define OMPTARGET_OPENMP_MAPPING_H
+
+#include "omptarget.h"
+
+#include <cstdint>
+#include <mutex>
+#include <string>
+
+#include "llvm/ADT/SmallSet.h"
+
+struct DeviceTy;
+class AsyncInfoTy;
+
+using map_var_info_t = void *;
+
+/// Information about shadow pointers.
+struct ShadowPtrInfoTy {
+ void **HstPtrAddr = nullptr;
+ void *HstPtrVal = nullptr;
+ void **TgtPtrAddr = nullptr;
+ void *TgtPtrVal = nullptr;
+
+ bool operator==(const ShadowPtrInfoTy &Other) const {
+ return HstPtrAddr == Other.HstPtrAddr;
+ }
+};
+
+inline bool operator<(const ShadowPtrInfoTy &lhs, const ShadowPtrInfoTy &rhs) {
+ return lhs.HstPtrAddr < rhs.HstPtrAddr;
+}
+
+/// Map between host data and target data.
+struct HostDataToTargetTy {
+ const uintptr_t HstPtrBase; // host info.
+ const uintptr_t HstPtrBegin;
+ const uintptr_t HstPtrEnd; // non-inclusive.
+ const map_var_info_t HstPtrName; // Optional source name of mapped variable.
+
+ const uintptr_t TgtAllocBegin; // allocated target memory
+ const uintptr_t TgtPtrBegin; // mapped target memory = TgtAllocBegin + padding
+
+private:
+ static const uint64_t INFRefCount = ~(uint64_t)0;
+ static std::string refCountToStr(uint64_t RefCount) {
+ return RefCount == INFRefCount ? "INF" : std::to_string(RefCount);
+ }
+
+ struct StatesTy {
+ StatesTy(uint64_t DRC, uint64_t HRC)
+ : DynRefCount(DRC), HoldRefCount(HRC) {}
+ /// The dynamic reference count is the standard reference count as of OpenMP
+ /// 4.5. The hold reference count is an OpenMP extension for the sake of
+ /// OpenACC support.
+ ///
+ /// The 'ompx_hold' map type modifier is permitted only on "omp target" and
+ /// "omp target data", and "delete" is permitted only on "omp target exit
+ /// data" and associated runtime library routines. As a result, we really
+ /// need to implement "reset" functionality only for the dynamic reference
+ /// counter. Likewise, only the dynamic reference count can be infinite
+ /// because, for example, omp_target_associate_ptr and "omp declare target
+ /// link" operate only on it. Nevertheless, it's actually easier to follow
+ /// the code (and requires less assertions for special cases) when we just
+ /// implement these features generally across both reference counters here.
+ /// Thus, it's the users of this class that impose those restrictions.
+ ///
+ uint64_t DynRefCount;
+ uint64_t HoldRefCount;
+
+ /// A map of shadow pointers associated with this entry, the keys are host
+ /// pointer addresses to identify stale entries.
+ llvm::SmallSet<ShadowPtrInfoTy, 2> ShadowPtrInfos;
+
+ /// Pointer to the event corresponding to the data update of this map.
+ /// Note: At present this event is created when the first data transfer from
+ /// host to device is issued, and only being used for H2D. It is not used
+ /// for data transfer in another direction (device to host). It is still
+ /// unclear whether we need it for D2H. If in the future we need similar
+ /// mechanism for D2H, and if the event cannot be shared between them, Event
+ /// should be written as <tt>void *Event[2]</tt>.
+ void *Event = nullptr;
+
+ /// Number of threads currently holding a reference to the entry at a
+ /// targetDataEnd. This is used to ensure that only the last thread that
+ /// references this entry will actually delete it.
+ int32_t DataEndThreadCount = 0;
+ };
+ // When HostDataToTargetTy is used by std::set, std::set::iterator is const
+ // use unique_ptr to make States mutable.
+ const std::unique_ptr<StatesTy> States;
+
+public:
+ HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E,
+ uintptr_t TgtAllocBegin, uintptr_t TgtPtrBegin,
+ bool UseHoldRefCount, map_var_info_t Name = nullptr,
+ bool IsINF = false)
+ : HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E), HstPtrName(Name),
+ TgtAllocBegin(TgtAllocBegin), TgtPtrBegin(TgtPtrBegin),
+ States(std::make_unique<StatesTy>(UseHoldRefCount ? 0
+ : IsINF ? INFRefCount
+ : 1,
+ !UseHoldRefCount ? 0
+ : IsINF ? INFRefCount
+ : 1)) {}
+
+ /// Get the total reference count. This is smarter than just getDynRefCount()
+ /// + getHoldRefCount() because it handles the case where at least one is
+ /// infinity and the other is non-zero.
+ uint64_t getTotalRefCount() const {
+ if (States->DynRefCount == INFRefCount ||
+ States->HoldRefCount == INFRefCount)
+ return INFRefCount;
+ return States->DynRefCount + States->HoldRefCount;
+ }
+
+ /// Get the dynamic reference count.
+ uint64_t getDynRefCount() const { return States->DynRefCount; }
+
+ /// Get the hold reference count.
+ uint64_t getHoldRefCount() const { return States->HoldRefCount; }
+
+ /// Get the event bound to this data map.
+ void *getEvent() const { return States->Event; }
+
+ /// Add a new event, if necessary.
+ /// Returns OFFLOAD_FAIL if something went wrong, OFFLOAD_SUCCESS otherwise.
+ int addEventIfNecessary(DeviceTy &Device, AsyncInfoTy &AsyncInfo) const;
+
+ /// Functions that manages the number of threads referencing the entry in a
+ /// targetDataEnd.
+ void incDataEndThreadCount() { ++States->DataEndThreadCount; }
+
+ [[nodiscard]] int32_t decDataEndThreadCount() {
+ return --States->DataEndThreadCount;
+ }
+
+ [[nodiscard]] int32_t getDataEndThreadCount() const {
+ return States->DataEndThreadCount;
+ }
+
+ /// Set the event bound to this data map.
+ void setEvent(void *Event) const { States->Event = Event; }
+
+ /// Reset the specified reference count unless it's infinity. Reset to 1
+ /// (even if currently 0) so it can be followed by a decrement.
+ void resetRefCount(bool UseHoldRefCount) const {
+ uint64_t &ThisRefCount =
+ UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
+ if (ThisRefCount != INFRefCount)
+ ThisRefCount = 1;
+ }
+
+ /// Increment the specified reference count unless it's infinity.
+ void incRefCount(bool UseHoldRefCount) const {
+ uint64_t &ThisRefCount =
+ UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
+ if (ThisRefCount != INFRefCount) {
+ ++ThisRefCount;
+ assert(ThisRefCount < INFRefCount && "refcount overflow");
+ }
+ }
+
+ /// Decrement the specified reference count unless it's infinity or zero, and
+ /// return the total reference count.
+ uint64_t decRefCount(bool UseHoldRefCount) const {
+ uint64_t &ThisRefCount =
+ UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
+ uint64_t OtherRefCount =
+ UseHoldRefCount ? States->DynRefCount : States->HoldRefCount;
+ (void)OtherRefCount;
+ if (ThisRefCount != INFRefCount) {
+ if (ThisRefCount > 0)
+ --ThisRefCount;
+ else
+ assert(OtherRefCount >= 0 && "total refcount underflow");
+ }
+ return getTotalRefCount();
+ }
+
+ /// Is the dynamic (and thus the total) reference count infinite?
+ bool isDynRefCountInf() const { return States->DynRefCount == INFRefCount; }
+
+ /// Convert the dynamic reference count to a debug string.
+ std::string dynRefCountToStr() const {
+ return refCountToStr(States->DynRefCount);
+ }
+
+ /// Convert the hold reference count to a debug string.
+ std::string holdRefCountToStr() const {
+ return refCountToStr(States->HoldRefCount);
+ }
+
+ /// Should one decrement of the specified reference count (after resetting it
+ /// if \c AfterReset) remove this mapping?
+ bool decShouldRemove(bool UseHoldRefCount, bool AfterReset = false) const {
+ uint64_t ThisRefCount =
+ UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
+ uint64_t OtherRefCount =
+ UseHoldRefCount ? States->DynRefCount : States->HoldRefCount;
+ if (OtherRefCount > 0)
+ return false;
+ if (AfterReset)
+ return ThisRefCount != INFRefCount;
+ return ThisRefCount == 1;
+ }
+
+ /// Add the shadow pointer info \p ShadowPtrInfo to this entry but only if the
+ /// the target ptr value was not already present in the existing set of shadow
+ /// pointers. Return true if something was added.
+ bool addShadowPointer(const ShadowPtrInfoTy &ShadowPtrInfo) const {
+ auto Pair = States->ShadowPtrInfos.insert(ShadowPtrInfo);
+ if (Pair.second)
+ return true;
+ // Check for a stale entry, if found, replace the old one.
+ if ((*Pair.first).TgtPtrVal == ShadowPtrInfo.TgtPtrVal)
+ return false;
+ States->ShadowPtrInfos.erase(ShadowPtrInfo);
+ return addShadowPointer(ShadowPtrInfo);
+ }
+
+ /// Apply \p CB to all shadow pointers of this entry. Returns OFFLOAD_FAIL if
+ /// \p CB returned OFFLOAD_FAIL for any of them, otherwise this returns
+ /// OFFLOAD_SUCCESS. The entry is locked for this operation.
+ template <typename CBTy> int foreachShadowPointerInfo(CBTy CB) const {
+ for (auto &It : States->ShadowPtrInfos)
+ if (CB(const_cast<ShadowPtrInfoTy &>(It)) == OFFLOAD_FAIL)
+ return OFFLOAD_FAIL;
+ return OFFLOAD_SUCCESS;
+ }
+
+ /// Lock this entry for exclusive access. Ensure to get exclusive access to
+ /// HDTTMap first!
+ void lock() const { Mtx.lock(); }
+
+ /// Unlock this entry to allow other threads inspecting it.
+ void unlock() const { Mtx.unlock(); }
+
+private:
+ // Mutex that needs to be held before the entry is inspected or modified. The
+ // HDTTMap mutex needs to be held before trying to lock any HDTT Entry.
+ mutable std::mutex Mtx;
+};
+
+/// Wrapper around the HostDataToTargetTy to be used in the HDTT map. In
+/// addition to the HDTT pointer we store the key value explicitly. This
+/// allows the set to inspect (sort/search/...) this entry without an additional
+/// load of HDTT. HDTT is a pointer to allow the modification of the set without
+/// invalidating HDTT entries which can now be inspected at the same time.
+struct HostDataToTargetMapKeyTy {
+ uintptr_t KeyValue;
+
+ HostDataToTargetMapKeyTy(void *Key) : KeyValue(uintptr_t(Key)) {}
+ HostDataToTargetMapKeyTy(uintptr_t Key) : KeyValue(Key) {}
+ HostDataToTargetMapKeyTy(HostDataToTargetTy *HDTT)
+ : KeyValue(HDTT->HstPtrBegin), HDTT(HDTT) {}
+ HostDataToTargetTy *HDTT;
+};
+inline bool operator<(const HostDataToTargetMapKeyTy &LHS,
+ const uintptr_t &RHS) {
+ return LHS.KeyValue < RHS;
+}
+inline bool operator<(const uintptr_t &LHS,
+ const HostDataToTargetMapKeyTy &RHS) {
+ return LHS < RHS.KeyValue;
+}
+inline bool operator<(const HostDataToTargetMapKeyTy &LHS,
+ const HostDataToTargetMapKeyTy &RHS) {
+ return LHS.KeyValue < RHS.KeyValue;
+}
+
+/// This struct will be returned by \p DeviceTy::getTargetPointer which provides
+/// more data than just a target pointer. A TargetPointerResultTy that has a non
+/// null Entry owns the entry. As long as the TargetPointerResultTy (TPR) exists
+/// the entry is locked. To give up ownership without destroying the TPR use the
+/// reset() function.
+struct TargetPointerResultTy {
+ struct FlagTy {
+ /// If the map table entry is just created
+ unsigned IsNewEntry : 1;
+ /// If the pointer is actually a host pointer (when unified memory enabled)
+ unsigned IsHostPointer : 1;
+ /// If the pointer is present in the mapping table.
+ unsigned IsPresent : 1;
+ /// Flag indicating that this was the last user of the entry and the ref
+ /// count is now 0.
+ unsigned IsLast : 1;
+ /// If the pointer is contained.
+ unsigned IsContained : 1;
+ } Flags = {0, 0, 0, 0, 0};
+
+ TargetPointerResultTy(const TargetPointerResultTy &) = delete;
+ TargetPointerResultTy &operator=(const TargetPointerResultTy &TPR) = delete;
+ TargetPointerResultTy() {}
+
+ TargetPointerResultTy(FlagTy Flags, HostDataToTargetTy *Entry,
+ void *TargetPointer)
+ : Flags(Flags), TargetPointer(TargetPointer), Entry(Entry) {
+ if (Entry)
+ Entry->lock();
+ }
+
+ TargetPointerResultTy(TargetPointerResultTy &&TPR)
+ : Flags(TPR.Flags), TargetPointer(TPR.TargetPointer), Entry(TPR.Entry) {
+ TPR.Entry = nullptr;
+ }
+
+ TargetPointerResultTy &operator=(TargetPointerResultTy &&TPR) {
+ if (&TPR != this) {
+ std::swap(Flags, TPR.Flags);
+ std::swap(Entry, TPR.Entry);
+ std::swap(TargetPointer, TPR.TargetPointer);
+ }
+ return *this;
+ }
+
+ ~TargetPointerResultTy() {
+ if (Entry)
+ Entry->unlock();
+ }
+
+ bool isPresent() const { return Flags.IsPresent; }
+
+ bool isHostPointer() const { return Flags.IsHostPointer; }
+
+ bool isContained() const { return Flags.IsContained; }
+
+ /// The corresponding target pointer
+ void *TargetPointer = nullptr;
+
+ HostDataToTargetTy *getEntry() const { return Entry; }
+ void setEntry(HostDataToTargetTy *HDTTT,
+ HostDataToTargetTy *OwnedTPR = nullptr) {
+ if (Entry)
+ Entry->unlock();
+ Entry = HDTTT;
+ if (Entry && Entry != OwnedTPR)
+ Entry->lock();
+ }
+
+ void reset() { *this = TargetPointerResultTy(); }
+
+private:
+ /// The corresponding map table entry which is stable.
+ HostDataToTargetTy *Entry = nullptr;
+};
+
+struct LookupResult {
+ struct {
+ unsigned IsContained : 1;
+ unsigned ExtendsBefore : 1;
+ unsigned ExtendsAfter : 1;
+ } Flags;
+
+ LookupResult() : Flags({0, 0, 0}), TPR() {}
+
+ TargetPointerResultTy TPR;
+};
+
+// This structure stores information of a mapped memory region.
+struct MapComponentInfoTy {
+ void *Base;
+ void *Begin;
+ int64_t Size;
+ int64_t Type;
+ void *Name;
+ MapComponentInfoTy() = default;
+ MapComponentInfoTy(void *Base, void *Begin, int64_t Size, int64_t Type,
+ void *Name)
+ : Base(Base), Begin(Begin), Size(Size), Type(Type), Name(Name) {}
+};
+
+// This structure stores all components of a user-defined mapper. The number of
+// components are dynamically decided, so we utilize C++ STL vector
+// implementation here.
+struct MapperComponentsTy {
+ llvm::SmallVector<MapComponentInfoTy> Components;
+ int32_t size() { return Components.size(); }
+};
+
+// The mapper function pointer type. It follows the signature below:
+// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
+// void *base, void *begin,
+// size_t size, int64_t type,
+// void * name);
+typedef void (*MapperFuncPtrTy)(void *, void *, void *, int64_t, int64_t,
+ void *);
+
+// Function pointer type for targetData* functions (targetDataBegin,
+// targetDataEnd and targetDataUpdate).
+typedef int (*TargetDataFuncPtrTy)(ident_t *, DeviceTy &, int32_t, void **,
+ void **, int64_t *, int64_t *,
+ map_var_info_t *, void **, AsyncInfoTy &,
+ bool);
+
+void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device);
+
+int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
+ void **ArgsBase, void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, map_var_info_t *ArgNames,
+ void **ArgMappers, AsyncInfoTy &AsyncInfo,
+ bool FromMapper = false);
+
+int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
+ void **ArgBases, void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, map_var_info_t *ArgNames,
+ void **ArgMappers, AsyncInfoTy &AsyncInfo,
+ bool FromMapper = false);
+
+int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
+ void **ArgsBase, void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, map_var_info_t *ArgNames,
+ void **ArgMappers, AsyncInfoTy &AsyncInfo,
+ bool FromMapper = false);
+
+#endif // OMPTARGET_OPENMP_MAPPING_H
diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h
index 74b59a4ab367c75..9cea6e6c9393012 100644
--- a/openmp/libomptarget/include/device.h
+++ b/openmp/libomptarget/include/device.h
@@ -19,15 +19,15 @@
#include <cstring>
#include <list>
#include <map>
-#include <memory>
#include <mutex>
#include <set>
-#include <thread>
#include "ExclusiveAccess.h"
#include "omptarget.h"
#include "rtl.h"
-#include "llvm/ADT/SmallSet.h"
+
+#include "OpenMP/Mapping.h"
+
#include "llvm/ADT/SmallVector.h"
// Forward declarations.
@@ -35,8 +35,6 @@ struct RTLInfoTy;
struct __tgt_bin_desc;
struct __tgt_target_table;
-using map_var_info_t = void *;
-
// enum for OMP_TARGET_OFFLOAD; keep in sync with kmp.h definition
enum kmp_target_offload_kind {
tgt_disabled = 0,
@@ -45,348 +43,6 @@ enum kmp_target_offload_kind {
};
typedef enum kmp_target_offload_kind kmp_target_offload_kind_t;
-/// Information about shadow pointers.
-struct ShadowPtrInfoTy {
- void **HstPtrAddr = nullptr;
- void *HstPtrVal = nullptr;
- void **TgtPtrAddr = nullptr;
- void *TgtPtrVal = nullptr;
-
- bool operator==(const ShadowPtrInfoTy &Other) const {
- return HstPtrAddr == Other.HstPtrAddr;
- }
-};
-
-inline bool operator<(const ShadowPtrInfoTy &lhs, const ShadowPtrInfoTy &rhs) {
- return lhs.HstPtrAddr < rhs.HstPtrAddr;
-}
-
-/// Map between host data and target data.
-struct HostDataToTargetTy {
- const uintptr_t HstPtrBase; // host info.
- const uintptr_t HstPtrBegin;
- const uintptr_t HstPtrEnd; // non-inclusive.
- const map_var_info_t HstPtrName; // Optional source name of mapped variable.
-
- const uintptr_t TgtAllocBegin; // allocated target memory
- const uintptr_t TgtPtrBegin; // mapped target memory = TgtAllocBegin + padding
-
-private:
- static const uint64_t INFRefCount = ~(uint64_t)0;
- static std::string refCountToStr(uint64_t RefCount) {
- return RefCount == INFRefCount ? "INF" : std::to_string(RefCount);
- }
-
- struct StatesTy {
- StatesTy(uint64_t DRC, uint64_t HRC)
- : DynRefCount(DRC), HoldRefCount(HRC) {}
- /// The dynamic reference count is the standard reference count as of OpenMP
- /// 4.5. The hold reference count is an OpenMP extension for the sake of
- /// OpenACC support.
- ///
- /// The 'ompx_hold' map type modifier is permitted only on "omp target" and
- /// "omp target data", and "delete" is permitted only on "omp target exit
- /// data" and associated runtime library routines. As a result, we really
- /// need to implement "reset" functionality only for the dynamic reference
- /// counter. Likewise, only the dynamic reference count can be infinite
- /// because, for example, omp_target_associate_ptr and "omp declare target
- /// link" operate only on it. Nevertheless, it's actually easier to follow
- /// the code (and requires less assertions for special cases) when we just
- /// implement these features generally across both reference counters here.
- /// Thus, it's the users of this class that impose those restrictions.
- ///
- uint64_t DynRefCount;
- uint64_t HoldRefCount;
-
- /// A map of shadow pointers associated with this entry, the keys are host
- /// pointer addresses to identify stale entries.
- llvm::SmallSet<ShadowPtrInfoTy, 2> ShadowPtrInfos;
-
- /// Pointer to the event corresponding to the data update of this map.
- /// Note: At present this event is created when the first data transfer from
- /// host to device is issued, and only being used for H2D. It is not used
- /// for data transfer in another direction (device to host). It is still
- /// unclear whether we need it for D2H. If in the future we need similar
- /// mechanism for D2H, and if the event cannot be shared between them, Event
- /// should be written as <tt>void *Event[2]</tt>.
- void *Event = nullptr;
-
- /// Number of threads currently holding a reference to the entry at a
- /// targetDataEnd. This is used to ensure that only the last thread that
- /// references this entry will actually delete it.
- int32_t DataEndThreadCount = 0;
- };
- // When HostDataToTargetTy is used by std::set, std::set::iterator is const
- // use unique_ptr to make States mutable.
- const std::unique_ptr<StatesTy> States;
-
-public:
- HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E,
- uintptr_t TgtAllocBegin, uintptr_t TgtPtrBegin,
- bool UseHoldRefCount, map_var_info_t Name = nullptr,
- bool IsINF = false)
- : HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E), HstPtrName(Name),
- TgtAllocBegin(TgtAllocBegin), TgtPtrBegin(TgtPtrBegin),
- States(std::make_unique<StatesTy>(UseHoldRefCount ? 0
- : IsINF ? INFRefCount
- : 1,
- !UseHoldRefCount ? 0
- : IsINF ? INFRefCount
- : 1)) {}
-
- /// Get the total reference count. This is smarter than just getDynRefCount()
- /// + getHoldRefCount() because it handles the case where at least one is
- /// infinity and the other is non-zero.
- uint64_t getTotalRefCount() const {
- if (States->DynRefCount == INFRefCount ||
- States->HoldRefCount == INFRefCount)
- return INFRefCount;
- return States->DynRefCount + States->HoldRefCount;
- }
-
- /// Get the dynamic reference count.
- uint64_t getDynRefCount() const { return States->DynRefCount; }
-
- /// Get the hold reference count.
- uint64_t getHoldRefCount() const { return States->HoldRefCount; }
-
- /// Get the event bound to this data map.
- void *getEvent() const { return States->Event; }
-
- /// Add a new event, if necessary.
- /// Returns OFFLOAD_FAIL if something went wrong, OFFLOAD_SUCCESS otherwise.
- int addEventIfNecessary(DeviceTy &Device, AsyncInfoTy &AsyncInfo) const;
-
- /// Functions that manages the number of threads referencing the entry in a
- /// targetDataEnd.
- void incDataEndThreadCount() { ++States->DataEndThreadCount; }
-
- [[nodiscard]] int32_t decDataEndThreadCount() {
- return --States->DataEndThreadCount;
- }
-
- [[nodiscard]] int32_t getDataEndThreadCount() const {
- return States->DataEndThreadCount;
- }
-
- /// Set the event bound to this data map.
- void setEvent(void *Event) const { States->Event = Event; }
-
- /// Reset the specified reference count unless it's infinity. Reset to 1
- /// (even if currently 0) so it can be followed by a decrement.
- void resetRefCount(bool UseHoldRefCount) const {
- uint64_t &ThisRefCount =
- UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
- if (ThisRefCount != INFRefCount)
- ThisRefCount = 1;
- }
-
- /// Increment the specified reference count unless it's infinity.
- void incRefCount(bool UseHoldRefCount) const {
- uint64_t &ThisRefCount =
- UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
- if (ThisRefCount != INFRefCount) {
- ++ThisRefCount;
- assert(ThisRefCount < INFRefCount && "refcount overflow");
- }
- }
-
- /// Decrement the specified reference count unless it's infinity or zero, and
- /// return the total reference count.
- uint64_t decRefCount(bool UseHoldRefCount) const {
- uint64_t &ThisRefCount =
- UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
- uint64_t OtherRefCount =
- UseHoldRefCount ? States->DynRefCount : States->HoldRefCount;
- (void)OtherRefCount;
- if (ThisRefCount != INFRefCount) {
- if (ThisRefCount > 0)
- --ThisRefCount;
- else
- assert(OtherRefCount >= 0 && "total refcount underflow");
- }
- return getTotalRefCount();
- }
-
- /// Is the dynamic (and thus the total) reference count infinite?
- bool isDynRefCountInf() const { return States->DynRefCount == INFRefCount; }
-
- /// Convert the dynamic reference count to a debug string.
- std::string dynRefCountToStr() const {
- return refCountToStr(States->DynRefCount);
- }
-
- /// Convert the hold reference count to a debug string.
- std::string holdRefCountToStr() const {
- return refCountToStr(States->HoldRefCount);
- }
-
- /// Should one decrement of the specified reference count (after resetting it
- /// if \c AfterReset) remove this mapping?
- bool decShouldRemove(bool UseHoldRefCount, bool AfterReset = false) const {
- uint64_t ThisRefCount =
- UseHoldRefCount ? States->HoldRefCount : States->DynRefCount;
- uint64_t OtherRefCount =
- UseHoldRefCount ? States->DynRefCount : States->HoldRefCount;
- if (OtherRefCount > 0)
- return false;
- if (AfterReset)
- return ThisRefCount != INFRefCount;
- return ThisRefCount == 1;
- }
-
- /// Add the shadow pointer info \p ShadowPtrInfo to this entry but only if the
- /// the target ptr value was not already present in the existing set of shadow
- /// pointers. Return true if something was added.
- bool addShadowPointer(const ShadowPtrInfoTy &ShadowPtrInfo) const {
- auto Pair = States->ShadowPtrInfos.insert(ShadowPtrInfo);
- if (Pair.second)
- return true;
- // Check for a stale entry, if found, replace the old one.
- if ((*Pair.first).TgtPtrVal == ShadowPtrInfo.TgtPtrVal)
- return false;
- States->ShadowPtrInfos.erase(ShadowPtrInfo);
- return addShadowPointer(ShadowPtrInfo);
- }
-
- /// Apply \p CB to all shadow pointers of this entry. Returns OFFLOAD_FAIL if
- /// \p CB returned OFFLOAD_FAIL for any of them, otherwise this returns
- /// OFFLOAD_SUCCESS. The entry is locked for this operation.
- template <typename CBTy> int foreachShadowPointerInfo(CBTy CB) const {
- for (auto &It : States->ShadowPtrInfos)
- if (CB(const_cast<ShadowPtrInfoTy &>(It)) == OFFLOAD_FAIL)
- return OFFLOAD_FAIL;
- return OFFLOAD_SUCCESS;
- }
-
- /// Lock this entry for exclusive access. Ensure to get exclusive access to
- /// HDTTMap first!
- void lock() const { Mtx.lock(); }
-
- /// Unlock this entry to allow other threads inspecting it.
- void unlock() const { Mtx.unlock(); }
-
-private:
- // Mutex that needs to be held before the entry is inspected or modified. The
- // HDTTMap mutex needs to be held before trying to lock any HDTT Entry.
- mutable std::mutex Mtx;
-};
-
-/// Wrapper around the HostDataToTargetTy to be used in the HDTT map. In
-/// addition to the HDTT pointer we store the key value explicitly. This
-/// allows the set to inspect (sort/search/...) this entry without an additional
-/// load of HDTT. HDTT is a pointer to allow the modification of the set without
-/// invalidating HDTT entries which can now be inspected at the same time.
-struct HostDataToTargetMapKeyTy {
- uintptr_t KeyValue;
-
- HostDataToTargetMapKeyTy(void *Key) : KeyValue(uintptr_t(Key)) {}
- HostDataToTargetMapKeyTy(uintptr_t Key) : KeyValue(Key) {}
- HostDataToTargetMapKeyTy(HostDataToTargetTy *HDTT)
- : KeyValue(HDTT->HstPtrBegin), HDTT(HDTT) {}
- HostDataToTargetTy *HDTT;
-};
-inline bool operator<(const HostDataToTargetMapKeyTy &LHS,
- const uintptr_t &RHS) {
- return LHS.KeyValue < RHS;
-}
-inline bool operator<(const uintptr_t &LHS,
- const HostDataToTargetMapKeyTy &RHS) {
- return LHS < RHS.KeyValue;
-}
-inline bool operator<(const HostDataToTargetMapKeyTy &LHS,
- const HostDataToTargetMapKeyTy &RHS) {
- return LHS.KeyValue < RHS.KeyValue;
-}
-
-/// This struct will be returned by \p DeviceTy::getTargetPointer which provides
-/// more data than just a target pointer. A TargetPointerResultTy that has a non
-/// null Entry owns the entry. As long as the TargetPointerResultTy (TPR) exists
-/// the entry is locked. To give up ownership without destroying the TPR use the
-/// reset() function.
-struct TargetPointerResultTy {
- struct FlagTy {
- /// If the map table entry is just created
- unsigned IsNewEntry : 1;
- /// If the pointer is actually a host pointer (when unified memory enabled)
- unsigned IsHostPointer : 1;
- /// If the pointer is present in the mapping table.
- unsigned IsPresent : 1;
- /// Flag indicating that this was the last user of the entry and the ref
- /// count is now 0.
- unsigned IsLast : 1;
- /// If the pointer is contained.
- unsigned IsContained : 1;
- } Flags = {0, 0, 0, 0, 0};
-
- TargetPointerResultTy(const TargetPointerResultTy &) = delete;
- TargetPointerResultTy &operator=(const TargetPointerResultTy &TPR) = delete;
- TargetPointerResultTy() {}
-
- TargetPointerResultTy(FlagTy Flags, HostDataToTargetTy *Entry,
- void *TargetPointer)
- : Flags(Flags), TargetPointer(TargetPointer), Entry(Entry) {
- if (Entry)
- Entry->lock();
- }
-
- TargetPointerResultTy(TargetPointerResultTy &&TPR)
- : Flags(TPR.Flags), TargetPointer(TPR.TargetPointer), Entry(TPR.Entry) {
- TPR.Entry = nullptr;
- }
-
- TargetPointerResultTy &operator=(TargetPointerResultTy &&TPR) {
- if (&TPR != this) {
- std::swap(Flags, TPR.Flags);
- std::swap(Entry, TPR.Entry);
- std::swap(TargetPointer, TPR.TargetPointer);
- }
- return *this;
- }
-
- ~TargetPointerResultTy() {
- if (Entry)
- Entry->unlock();
- }
-
- bool isPresent() const { return Flags.IsPresent; }
-
- bool isHostPointer() const { return Flags.IsHostPointer; }
-
- bool isContained() const { return Flags.IsContained; }
-
- /// The corresponding target pointer
- void *TargetPointer = nullptr;
-
- HostDataToTargetTy *getEntry() const { return Entry; }
- void setEntry(HostDataToTargetTy *HDTTT,
- HostDataToTargetTy *OwnedTPR = nullptr) {
- if (Entry)
- Entry->unlock();
- Entry = HDTTT;
- if (Entry && Entry != OwnedTPR)
- Entry->lock();
- }
-
- void reset() { *this = TargetPointerResultTy(); }
-
-private:
- /// The corresponding map table entry which is stable.
- HostDataToTargetTy *Entry = nullptr;
-};
-
-struct LookupResult {
- struct {
- unsigned IsContained : 1;
- unsigned ExtendsBefore : 1;
- unsigned ExtendsAfter : 1;
- } Flags;
-
- LookupResult() : Flags({0, 0, 0}), TPR() {}
-
- TargetPointerResultTy TPR;
-};
-
///
struct PendingCtorDtorListsTy {
std::list<void *> PendingCtors;
diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt
index a83965f59b3eb04..bd085d99c6523cd 100644
--- a/openmp/libomptarget/src/CMakeLists.txt
+++ b/openmp/libomptarget/src/CMakeLists.txt
@@ -22,7 +22,9 @@ add_llvm_library(omptarget
OmptCallback.cpp
rtl.cpp
LegacyAPI.cpp
+
OpenMP/InteropAPI.cpp
+ OpenMP/Mapping.cpp
ADDITIONAL_HEADER_DIRS
${LIBOMPTARGET_INCLUDE_DIR}
diff --git a/openmp/libomptarget/src/OpenMP/Mapping.cpp b/openmp/libomptarget/src/OpenMP/Mapping.cpp
new file mode 100644
index 000000000000000..c7cb5b6b1e4c45d
--- /dev/null
+++ b/openmp/libomptarget/src/OpenMP/Mapping.cpp
@@ -0,0 +1,40 @@
+//===-- OpenMP/Mapping.cpp - OpenMP/OpenACC pointer mapping impl. ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "OpenMP/Mapping.h"
+
+#include "Shared/Debug.h"
+#include "device.h"
+
+/// Dump a table of all the host-target pointer pairs on failure
+void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) {
+ DeviceTy::HDTTMapAccessorTy HDTTMap =
+ Device.HostDataToTargetMap.getExclusiveAccessor();
+ if (HDTTMap->empty())
+ return;
+
+ SourceInfo Kernel(Loc);
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
+ "OpenMP Host-Device pointer mappings after block at %s:%d:%d:\n",
+ Kernel.getFilename(), Kernel.getLine(), Kernel.getColumn());
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID, "%-18s %-18s %s %s %s %s\n",
+ "Host Ptr", "Target Ptr", "Size (B)", "DynRefCount", "HoldRefCount",
+ "Declaration");
+ for (const auto &It : *HDTTMap) {
+ HostDataToTargetTy &HDTT = *It.HDTT;
+ SourceInfo Info(HDTT.HstPtrName);
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
+ DPxMOD " " DPxMOD " %-8" PRIuPTR " %-11s %-12s %s at %s:%d:%d\n",
+ DPxPTR(HDTT.HstPtrBegin), DPxPTR(HDTT.TgtPtrBegin),
+ HDTT.HstPtrEnd - HDTT.HstPtrBegin, HDTT.dynRefCountToStr().c_str(),
+ HDTT.holdRefCountToStr().c_str(), Info.getName(), Info.getFilename(),
+ Info.getLine(), Info.getColumn());
+ }
+}
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 96c27e417e79a5e..f549afe24f1ae94 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -20,6 +20,7 @@
#include "Shared/Profile.h"
+#include "OpenMP/Mapping.h"
#include "OpenMP/omp.h"
#include "llvm/ADT/StringExtras.h"
diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index 0730a1ea0fd4b44..b2d78f366dd5787 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -23,24 +23,6 @@
#include <cstdint>
-extern int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
- void **ArgsBase, void **Args, int64_t *ArgSizes,
- int64_t *ArgTypes, map_var_info_t *ArgNames,
- void **ArgMappers, AsyncInfoTy &AsyncInfo,
- bool FromMapper = false);
-
-extern int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
- void **ArgBases, void **Args, int64_t *ArgSizes,
- int64_t *ArgTypes, map_var_info_t *ArgNames,
- void **ArgMappers, AsyncInfoTy &AsyncInfo,
- bool FromMapper = false);
-
-extern int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
- void **ArgsBase, void **Args, int64_t *ArgSizes,
- int64_t *ArgTypes, map_var_info_t *ArgNames,
- void **ArgMappers, AsyncInfoTy &AsyncInfo,
- bool FromMapper = false);
-
extern int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
KernelArgsTy &KernelArgs, AsyncInfoTy &AsyncInfo);
@@ -65,42 +47,6 @@ extern void *targetLockExplicit(void *HostPtr, size_t Size, int DeviceNum,
extern void targetUnlockExplicit(void *HostPtr, int DeviceNum,
const char *Name);
-// This structure stores information of a mapped memory region.
-struct MapComponentInfoTy {
- void *Base;
- void *Begin;
- int64_t Size;
- int64_t Type;
- void *Name;
- MapComponentInfoTy() = default;
- MapComponentInfoTy(void *Base, void *Begin, int64_t Size, int64_t Type,
- void *Name)
- : Base(Base), Begin(Begin), Size(Size), Type(Type), Name(Name) {}
-};
-
-// This structure stores all components of a user-defined mapper. The number of
-// components are dynamically decided, so we utilize C++ STL vector
-// implementation here.
-struct MapperComponentsTy {
- llvm::SmallVector<MapComponentInfoTy> Components;
- int32_t size() { return Components.size(); }
-};
-
-// The mapper function pointer type. It follows the signature below:
-// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
-// void *base, void *begin,
-// size_t size, int64_t type,
-// void * name);
-typedef void (*MapperFuncPtrTy)(void *, void *, void *, int64_t, int64_t,
- void *);
-
-// Function pointer type for targetData* functions (targetDataBegin,
-// targetDataEnd and targetDataUpdate).
-typedef int (*TargetDataFuncPtrTy)(ident_t *, DeviceTy &, int32_t, void **,
- void **, int64_t *, int64_t *,
- map_var_info_t *, void **, AsyncInfoTy &,
- bool);
-
// Implemented in libomp, they are called from within __tgt_* functions.
#ifdef __cplusplus
extern "C" {
@@ -257,34 +203,6 @@ struct TargetMemsetArgsTy {
}
#endif
-////////////////////////////////////////////////////////////////////////////////
-/// dump a table of all the host-target pointer pairs on failure
-static inline void dumpTargetPointerMappings(const ident_t *Loc,
- DeviceTy &Device) {
- DeviceTy::HDTTMapAccessorTy HDTTMap =
- Device.HostDataToTargetMap.getExclusiveAccessor();
- if (HDTTMap->empty())
- return;
-
- SourceInfo Kernel(Loc);
- INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
- "OpenMP Host-Device pointer mappings after block at %s:%d:%d:\n",
- Kernel.getFilename(), Kernel.getLine(), Kernel.getColumn());
- INFO(OMP_INFOTYPE_ALL, Device.DeviceID, "%-18s %-18s %s %s %s %s\n",
- "Host Ptr", "Target Ptr", "Size (B)", "DynRefCount", "HoldRefCount",
- "Declaration");
- for (const auto &It : *HDTTMap) {
- HostDataToTargetTy &HDTT = *It.HDTT;
- SourceInfo Info(HDTT.HstPtrName);
- INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
- DPxMOD " " DPxMOD " %-8" PRIuPTR " %-11s %-12s %s at %s:%d:%d\n",
- DPxPTR(HDTT.HstPtrBegin), DPxPTR(HDTT.TgtPtrBegin),
- HDTT.HstPtrEnd - HDTT.HstPtrBegin, HDTT.dynRefCountToStr().c_str(),
- HDTT.holdRefCountToStr().c_str(), Info.getName(), Info.getFilename(),
- Info.getLine(), Info.getColumn());
- }
-}
-
////////////////////////////////////////////////////////////////////////////////
/// Print out the names and properties of the arguments to each kernel
static inline void
More information about the flang-commits
mailing list