[clang] [flang] [llvm] [DRAFT] [offload] Add new libacctarget runtime (PR #197894)
Ivan R. Ivanov via cfe-commits
cfe-commits at lists.llvm.org
Tue Nov 3 01:55:28 PST 2026
https://github.com/ivanradanov updated https://github.com/llvm/llvm-project/pull/197894
>From 2094f815e2a2acb289d36208c5fb81aa083f401f Mon Sep 17 00:00:00 2001
From: Ivan Radanov Ivanov <iivanov at nvidia.com>
Date: Wed, 2 Sep 2026 05:36:44 -0700
Subject: [PATCH] [acc] Add OpenACC support
---
.../ClangLinkerWrapper.cpp | 9 +-
flang-rt/lib/runtime/CMakeLists.txt | 1 +
flang/include/flang/Common/api-attrs.h | 6 +-
llvm/CMakeLists.txt | 2 +-
.../llvm/Frontend/Offloading/OffloadWrapper.h | 12 +-
llvm/include/llvm/Object/OffloadBinary.h | 4 +-
.../Frontend/Offloading/OffloadWrapper.cpp | 44 +-
llvm/lib/Object/OffloadBinary.cpp | 3 +
.../llvm-offload-wrapper/offload-wrapper.ll | 16 +
.../llvm-offload-wrapper.cpp | 5 +-
offload/CMakeLists.txt | 25 +-
offload/include/OpenMP/Mapping.h | 14 +-
offload/include/PluginManager.h | 22 +-
offload/include/Shared/Debug.h | 47 +-
offload/include/device.h | 30 +
offload/include/omptarget.h | 97 +-
offload/libacctarget/CMakeLists.txt | 71 +
.../libacctarget/CfiDataRuntimeInterface.cpp | 297 ++
offload/libacctarget/DataRuntimeInterface.cpp | 204 ++
offload/libacctarget/Debug.h | 26 +
offload/libacctarget/DeviceManager.cpp | 232 ++
offload/libacctarget/DeviceManager.h | 75 +
offload/libacctarget/Interface.cpp | 2427 +++++++++++++++++
offload/libacctarget/Interface.h | 301 ++
offload/libacctarget/Logger.h | 47 +
offload/libacctarget/Private.h | 29 +
offload/libacctarget/QueueManager.cpp | 180 ++
offload/libacctarget/QueueManager.h | 80 +
offload/libacctarget/RuntimeImpl.cpp | 175 ++
offload/libacctarget/RuntimeInterface.cpp | 249 ++
offload/libacctarget/RuntimeState.cpp | 80 +
offload/libacctarget/RuntimeState.h | 24 +
offload/libacctarget/exports | 191 ++
offload/libacctarget/include/openacc.h | 46 +
offload/libompaccsupport/AsyncInfo.cpp | 72 +-
offload/libompaccsupport/CMakeLists.txt | 9 +
offload/libompaccsupport/Mapping.cpp | 74 +-
offload/libompaccsupport/PluginManager.cpp | 61 +-
offload/libompaccsupport/device.cpp | 122 +-
offload/libomptarget/CMakeLists.txt | 8 +-
offload/libomptarget/interface.cpp | 34 +-
offload/libomptarget/omptarget.cpp | 43 +-
offload/test/acc/init.c | 22 +
offload/test/lit.cfg | 10 +
offload/test/lit.site.cfg.in | 1 +
openacc/CMakeLists.txt | 40 +
openacc/device/CMakeLists.txt | 82 +
openacc/device/src/AccEntryImpl.cpp | 23 +
runtimes/CMakeLists.txt | 2 +-
49 files changed, 5528 insertions(+), 146 deletions(-)
create mode 100644 offload/libacctarget/CMakeLists.txt
create mode 100644 offload/libacctarget/CfiDataRuntimeInterface.cpp
create mode 100644 offload/libacctarget/DataRuntimeInterface.cpp
create mode 100644 offload/libacctarget/Debug.h
create mode 100644 offload/libacctarget/DeviceManager.cpp
create mode 100644 offload/libacctarget/DeviceManager.h
create mode 100644 offload/libacctarget/Interface.cpp
create mode 100644 offload/libacctarget/Interface.h
create mode 100644 offload/libacctarget/Logger.h
create mode 100644 offload/libacctarget/Private.h
create mode 100644 offload/libacctarget/QueueManager.cpp
create mode 100644 offload/libacctarget/QueueManager.h
create mode 100644 offload/libacctarget/RuntimeImpl.cpp
create mode 100644 offload/libacctarget/RuntimeInterface.cpp
create mode 100644 offload/libacctarget/RuntimeState.cpp
create mode 100644 offload/libacctarget/RuntimeState.h
create mode 100644 offload/libacctarget/exports
create mode 100644 offload/libacctarget/include/openacc.h
create mode 100644 offload/libompaccsupport/CMakeLists.txt
create mode 100644 offload/test/acc/init.c
create mode 100644 openacc/CMakeLists.txt
create mode 100644 openacc/device/CMakeLists.txt
create mode 100644 openacc/device/src/AccEntryImpl.cpp
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 010335fd9fd37e..bf44f9e2d69541 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -802,10 +802,11 @@ wrapDeviceImages(ArrayRef<std::unique_ptr<MemoryBuffer>> Buffers,
M.setDataLayout(TheTriple.computeDataLayout());
switch (Kind) {
+ case OFK_OpenACC:
case OFK_OpenMP:
- if (Error Err = offloading::wrapOpenMPBinaries(
+ if (Error Err = offloading::wrapOpenMPOpenACCBinaries(
M, BuffersToWrap, offloading::getOffloadEntryArray(M),
- /*Suffix=*/"", /*Relocatable=*/Args.hasArg(OPT_relocatable)))
+ /*Suffix=*/"", /*Relocatable=*/Args.hasArg(OPT_relocatable), Kind))
return std::move(Err);
break;
case OFK_Cuda:
@@ -985,8 +986,10 @@ bundleLinkedOutput(ArrayRef<OffloadingImage> Images, const ArgList &Args,
switch (Kind) {
case OFK_OpenMP:
+ case OFK_OpenACC:
return (Verbose && SaveTemps) ? bundleOpenMPVerbose(Images)
: bundleOpenMP(Images);
+ return bundleOpenMP(Images);
case OFK_SYCL:
return bundleSYCL(Images);
case OFK_Cuda:
@@ -1152,7 +1155,7 @@ linkAndWrapDeviceFiles(ArrayRef<SmallVector<OffloadFile>> LinkerInputFiles,
return OutputOrErr.takeError();
// Store the offloading image for each linked output file.
- for (OffloadKind Kind = OFK_OpenMP; Kind != OFK_LAST;
+ for (OffloadKind Kind = OFK_FIRST; Kind != OFK_LAST;
Kind = static_cast<OffloadKind>((uint16_t)(Kind) << 1)) {
if ((ActiveOffloadKindMask & Kind) == 0)
continue;
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index f910f88932ac8a..466702383d5133 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -199,6 +199,7 @@ set(gpu_sources
reduction.cpp
temporary-stack.cpp
io-api-gpu.cpp
+ stl-overrides.cpp
)
file(GLOB_RECURSE public_headers
diff --git a/flang/include/flang/Common/api-attrs.h b/flang/include/flang/Common/api-attrs.h
index 76e7a2dcc7e931..7c0d9ac6d469b7 100644
--- a/flang/include/flang/Common/api-attrs.h
+++ b/flang/include/flang/Common/api-attrs.h
@@ -127,7 +127,7 @@
* separate host and device compilations.
*/
#if ((defined(__CUDACC__) || defined(__CUDA__)) && defined(__CUDA_ARCH__)) || \
- (defined(_OPENMP) && (defined(__AMDGCN__) || defined(__NVPTX__)))
+ (defined(__AMDGCN__) || defined(__NVPTX__))
#define RT_DEVICE_COMPILATION 1
#else
#undef RT_DEVICE_COMPILATION
@@ -152,8 +152,10 @@
* of a clang CUDA build, and the regular CUDA runtime library does compile
* them, hence the RT_DEVICE_COMPILATION check. The CUDA PTX library leaves
* them out and sets RT_THIN_IO itself from CMake.
+ * RT_DEVICE_COMPILATION is also set for native GPU builds, so it cannot be used
+ * to tell them apart from the CUDA and OpenMP device passes.
*/
-#if RT_GPU_TARGET && !defined(RT_DEVICE_COMPILATION)
+#if RT_GPU_TARGET && !defined(__CUDA_ARCH__) && !defined(_OPENMP)
#define RT_THIN_IO 1
#endif
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 4a3e307e4ab0a3..0b8978a5ad6158 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -175,7 +175,7 @@ set(LLVM_DEPENDENCY_ONLY_PROJECTS)
# As we migrate runtimes to using the bootstrapping build, the set of default runtimes
# should grow as we remove those runtimes from LLVM_ENABLE_PROJECTS above.
set(LLVM_DEFAULT_RUNTIMES "libcxx;libcxxabi;libunwind;libclc;compiler-rt;openmp")
-set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt;libclc;libsycl;orc-rt")
+set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;openacc;llvm-libgcc;offload;flang-rt;libclc;libsycl;orc-rt")
set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
"Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
if(LLVM_ENABLE_RUNTIMES STREQUAL "all")
diff --git a/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h b/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h
index 8d0c98f12b24de..ec1341748c891d 100644
--- a/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h
+++ b/llvm/include/llvm/Frontend/Offloading/OffloadWrapper.h
@@ -11,6 +11,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/IR/Module.h"
+#include "llvm/Object/OffloadBinary.h"
#include "llvm/Support/Compiler.h"
#include <string>
@@ -19,16 +20,17 @@ namespace llvm {
namespace offloading {
using EntryArrayTy = std::pair<Constant *, Constant *>;
/// Wraps the input device images into the module \p M as global symbols and
-/// registers the images with the OpenMP Offloading runtime libomptarget.
+/// registers the images with the OpenMP or OpenACC Offloading runtime
+/// libomptarget or libacctarget.
/// \param EntryArray Optional pair pointing to the begin and end of the
/// `__tgt_offload_entry` array.
/// \param Suffix An optional suffix appended to the emitted symbols.
/// \param Relocatable Indicate if we need to change the offloading section to
/// create a relocatable object.
-LLVM_ABI llvm::Error
-wrapOpenMPBinaries(llvm::Module &M, llvm::ArrayRef<llvm::ArrayRef<char>> Images,
- EntryArrayTy EntryArray, llvm::StringRef Suffix = "",
- bool Relocatable = false);
+LLVM_ABI llvm::Error wrapOpenMPOpenACCBinaries(
+ llvm::Module &M, llvm::ArrayRef<llvm::ArrayRef<char>> Images,
+ EntryArrayTy EntryArray, llvm::StringRef Suffix = "",
+ bool Relocatable = false, object::OffloadKind Kind = object::OFK_OpenMP);
/// Wraps the input fatbinary image into the module \p M as global symbols and
/// registers the images with the CUDA runtime.
diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h
index 9fbe4ad949c2a1..94928bf8af1709 100644
--- a/llvm/include/llvm/Object/OffloadBinary.h
+++ b/llvm/include/llvm/Object/OffloadBinary.h
@@ -35,11 +35,13 @@ namespace object {
/// The producer of the associated offloading image.
enum OffloadKind : uint16_t {
OFK_None = 0,
+ OFK_FIRST = (1 << 0),
OFK_OpenMP = (1 << 0),
OFK_Cuda = (1 << 1),
OFK_HIP = (1 << 2),
OFK_SYCL = (1 << 3),
- OFK_LAST = (1 << 4),
+ OFK_OpenACC = (1 << 4),
+ OFK_LAST = (1 << 5),
};
/// The type of contents the offloading image contains.
diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
index bcd65513bc8428..047ca0d65f121b 100644
--- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
+++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
@@ -197,7 +197,19 @@ GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
}
Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
- StringRef Suffix) {
+ StringRef Suffix, OffloadKind Kind) {
+ StringRef RegFuncName;
+ switch (Kind) {
+ case OFK_OpenMP:
+ RegFuncName = "__tgt_unregister_lib";
+ break;
+ case OFK_OpenACC:
+ RegFuncName = "__tgt_acc_unregister_lib";
+ break;
+ default:
+ llvm_unreachable("Unexpected Kind - only OpenMP or OpenACC allowed");
+ }
+
LLVMContext &C = M.getContext();
auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
auto *Func =
@@ -205,11 +217,11 @@ Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
".omp_offloading.descriptor_unreg" + Suffix, &M);
Func->setSection(getStartupSection(M.getTargetTriple()));
- // Get __tgt_unregister_lib function declaration.
+ // Get the runtime-specific unregister function declaration.
auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),
/*isVarArg*/ false);
FunctionCallee UnRegFuncC =
- M.getOrInsertFunction("__tgt_unregister_lib", UnRegFuncTy);
+ M.getOrInsertFunction(RegFuncName, UnRegFuncTy);
// Construct function body
IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
@@ -220,7 +232,7 @@ Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
}
void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
- StringRef Suffix) {
+ StringRef Suffix, OffloadKind Kind) {
LLVMContext &C = M.getContext();
auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
auto *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage,
@@ -230,14 +242,24 @@ void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
// Get __tgt_register_lib function declaration.
auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),
/*isVarArg*/ false);
- FunctionCallee RegFuncC =
- M.getOrInsertFunction("__tgt_register_lib", RegFuncTy);
+ StringRef RegFuncName;
+ switch (Kind) {
+ case OFK_OpenMP:
+ RegFuncName = "__tgt_register_lib";
+ break;
+ case OFK_OpenACC:
+ RegFuncName = "__tgt_acc_register_lib";
+ break;
+ default:
+ llvm_unreachable("Unexpected Kind - only OpenMP or OpenACC allowed");
+ }
+ FunctionCallee RegFuncC = M.getOrInsertFunction(RegFuncName, RegFuncTy);
auto *AtExitTy = FunctionType::get(
Type::getInt32Ty(C), PointerType::getUnqual(C), /*isVarArg=*/false);
FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy);
- Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);
+ Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix, Kind);
// Construct function body
IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
@@ -712,15 +734,15 @@ class SYCLWrapper {
} // namespace
-Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
- EntryArrayTy EntryArray,
- llvm::StringRef Suffix, bool Relocatable) {
+Error offloading::wrapOpenMPOpenACCBinaries(
+ Module &M, ArrayRef<ArrayRef<char>> Images, EntryArrayTy EntryArray,
+ llvm::StringRef Suffix, bool Relocatable, OffloadKind Kind) {
GlobalVariable *Desc =
createBinDesc(M, Images, EntryArray, Suffix, Relocatable);
if (!Desc)
return createStringError(inconvertibleErrorCode(),
"No binary descriptors created.");
- createRegisterFunction(M, Desc, Suffix);
+ createRegisterFunction(M, Desc, Suffix, Kind);
return Error::success();
}
diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp
index 7b9b9a416c35da..4adb089388daef 100644
--- a/llvm/lib/Object/OffloadBinary.cpp
+++ b/llvm/lib/Object/OffloadBinary.cpp
@@ -481,6 +481,7 @@ OffloadKind object::getOffloadKind(StringRef Name) {
.Case("cuda", OFK_Cuda)
.Case("hip", OFK_HIP)
.Case("sycl", OFK_SYCL)
+ .Case("openacc", OFK_OpenACC)
.Default(OFK_None);
}
@@ -494,6 +495,8 @@ StringRef object::getOffloadKindName(OffloadKind Kind) {
return "hip";
case OFK_SYCL:
return "sycl";
+ case OFK_OpenACC:
+ return "openacc";
default:
return "none";
}
diff --git a/llvm/test/tools/llvm-offload-wrapper/offload-wrapper.ll b/llvm/test/tools/llvm-offload-wrapper/offload-wrapper.ll
index 9d8abaca5924cb..46a9db2e398ffc 100644
--- a/llvm/test/tools/llvm-offload-wrapper/offload-wrapper.ll
+++ b/llvm/test/tools/llvm-offload-wrapper/offload-wrapper.ll
@@ -28,6 +28,22 @@
; OMP-NEXT: ret void
; OMP-NEXT: }
+; RUN: llvm-offload-wrapper --triple=x86_64-unknown-linux-gnu -kind=openacc %s -o %t.bc
+; RUN: llvm-dis %t.bc -o - | FileCheck %s --check-prefix=ACC
+
+; ACC: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
+; ACC-NEXT: entry:
+; ACC-NEXT: call void @__tgt_acc_register_lib(ptr @.omp_offloading.descriptor)
+; ACC-NEXT: %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
+; ACC-NEXT: ret void
+; ACC-NEXT: }
+
+; ACC: define internal void @.omp_offloading.descriptor_unreg() section ".text.startup" {
+; ACC-NEXT: entry:
+; ACC-NEXT: call void @__tgt_acc_unregister_lib(ptr @.omp_offloading.descriptor)
+; ACC-NEXT: ret void
+; ACC-NEXT: }
+
; RUN: llvm-offload-wrapper --triple=x86_64-unknown-linux-gnu -kind=hip %s -o %t.bc
; RUN: llvm-dis %t.bc -o - | FileCheck %s --check-prefix=HIP
diff --git a/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp b/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp
index ff5332ada15b4c..792fe0db33aa5a 100644
--- a/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp
+++ b/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp
@@ -72,9 +72,10 @@ static Error wrapImages(ArrayRef<ArrayRef<char>> BuffersToWrap) {
switch (Kind) {
case llvm::object::OFK_OpenMP:
- if (Error Err = offloading::wrapOpenMPBinaries(
+ case llvm::object::OFK_OpenACC:
+ if (Error Err = offloading::wrapOpenMPOpenACCBinaries(
M, BuffersToWrap, offloading::getOffloadEntryArray(M),
- /*Suffix=*/"", /*Relocatable=*/Relocatable))
+ /*Suffix=*/"", /*Relocatable=*/Relocatable, Kind))
return Err;
break;
case llvm::object::OFK_Cuda:
diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index e09ba60c5055e8..cf77e8d3090f69 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -16,26 +16,28 @@ set(LLVM_SUBPROJECT_TITLE "liboffload")
set(OFFLOAD_MIN_CUDA_VERSION 11.8.0)
set(OFFLOAD_MIN_CUDA_VERSION_CODE 11080)
-# libomptarget is not supported on Windows.
-set(BUILD_LIBOMPTARGET ON)
+option(BUILD_LIBOMPTARGET "Build the OpenMP offloading runtime library libomptarget." ON)
+option(BUILD_LIBACCTARGET "Build the OpenACC offloading runtime library libacctarget." ON)
+# libomptarget and libacctarget are not supported on Windows.
if(MSVC)
- set(BUILD_LIBOMPTARGET OFF)
- message(STATUS "libomptarget is not supported on Windows, skipping.")
+ set(BUILD_LIBOMPTARGET OFF CACHE BOOL "" FORCE)
+ set(BUILD_LIBACCTARGET OFF CACHE BOOL "" FORCE)
+ message(STATUS "libomptarget and libacctarget are not supported on Windows, skipping.")
endif()
# Check that the library can actually be built.
if(APPLE OR WASM)
- message(WARNING "libomptarget cannot be built on MacOS X and WASM!")
+ message(WARNING "offload cannot be built on MacOS X and WASM!")
return()
elseif("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^(amdgpu|amdgcn|nvptx|spirv)" OR
"${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^(amdgpu|amdgcn|nvptx|spirv)")
message(WARNING "offload cannot be built on GPU targets yet.")
return()
elseif(NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES)
- message(WARNING "Host compiler must support C++17 to build libomptarget!")
+ message(WARNING "Host compiler must support C++17 to build offload!")
return()
elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
- message(WARNING "libomptarget on 32-bit systems is not supported!")
+ message(WARNING "offload on 32-bit systems is not supported!")
return()
endif()
@@ -317,6 +319,7 @@ else()
endif()
pythonize_bool(LIBOMPTARGET_OMPT_SUPPORT)
+pythonize_bool(BUILD_LIBACCTARGET)
set(LIBOMPTARGET_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
set(LIBOMPTARGET_BINARY_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/include)
@@ -364,10 +367,16 @@ add_subdirectory(docs)
add_subdirectory(liboffload)
+add_subdirectory(libompaccsupport)
+
if(BUILD_LIBOMPTARGET)
add_subdirectory(libomptarget)
endif()
+if(BUILD_LIBACCTARGET)
+ add_subdirectory(libacctarget)
+endif()
+
add_subdirectory(languages)
# Add tests.
@@ -376,7 +385,7 @@ if(OFFLOAD_INCLUDE_TESTS)
add_subdirectory(unittests)
endif()
-foreach(target omptarget LLVMOffload LLVMOffloadKernel)
+foreach(target omptarget acctarget LLVMOffload LLVMOffloadKernel)
if(TARGET ${target})
add_dependencies(offload ${target})
endif()
diff --git a/offload/include/OpenMP/Mapping.h b/offload/include/OpenMP/Mapping.h
index 4b160684ddb327..570f9ab204d23e 100644
--- a/offload/include/OpenMP/Mapping.h
+++ b/offload/include/OpenMP/Mapping.h
@@ -20,6 +20,7 @@
#include <cstdint>
#include <mutex>
#include <string>
+#include <variant>
#include "llvm/ADT/SmallSet.h"
@@ -667,9 +668,11 @@ struct MappingInfoTy {
/// - Data transfer issue fails.
TargetPointerResultTy getTargetPointer(
HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
- int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
- bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
- bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+ int64_t TgtPadding,
+ std::variant<int64_t, const NonContigDescTy *> MemInfo,
+ map_var_info_t HstPtrName, bool HasFlagTo, bool HasFlagAlways,
+ bool IsImplicit, bool UpdateRefCount, bool HasCloseModifier,
+ bool HasPresentModifier, bool HasHoldModifier, bool IsNoCreate,
AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
bool ReleaseHDTTMap = true, StateInfoTy *StateInfo = nullptr,
const ident_t *Loc = nullptr);
@@ -713,6 +716,11 @@ struct MappingInfoTy {
int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
int disassociatePtr(void *HstPtrBegin);
+ void printNonContigCopyInfo(void *TgtPtrBegin, void *HstPtrBegin,
+ const NonContigDescTy &CopyInfo, bool H2D,
+ HostDataToTargetTy *Entry,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr);
+
/// Print information about the transfer from \p HstPtr to \p TgtPtr (or vice
/// versa if \p H2D is false). If there is an existing mapping, or if \p Entry
/// is set, the associated metadata will be printed as well.
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index ab06e13f331a50..48e7224ebbc702 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -32,6 +32,7 @@
#include "llvm/Support/Error.h"
#include <cstdint>
+#include <functional>
#include <list>
#include <memory>
#include <mutex>
@@ -93,18 +94,23 @@ struct PluginManager {
// Work around for plugins that call dlopen on shared libraries that call
// tgt_register_lib during their initialisation. Stash the pointers in a
// vector until the plugins are all initialised and then register them.
- bool delayRegisterLib(__tgt_bin_desc *Desc) {
+ /// Find the table information in the map or look it up in the translation
+ /// tables.
+ TableMap *getTableMap(void *HostPtr);
+
+ bool delayRegisterLib(std::function<void(__tgt_bin_desc *)> RegisterFunc,
+ __tgt_bin_desc *Desc) {
if (RTLsLoaded)
return false;
- DelayedBinDesc.push_back(Desc);
+ DelayedBinDesc.push_back({RegisterFunc, Desc});
return true;
}
void registerDelayedLibraries() {
// Only called by libomptarget constructor
RTLsLoaded = true;
- for (auto *Desc : DelayedBinDesc)
- __tgt_register_lib(Desc);
+ for (auto &[RegisterFunc, Desc] : DelayedBinDesc)
+ RegisterFunc(Desc);
DelayedBinDesc.clear();
}
@@ -148,7 +154,9 @@ struct PluginManager {
bool RTLsLoaded = false;
// Whether init() called olInit, so deinit() must call olShutDown.
bool LibOffloadInitialized = false;
- llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
+ llvm::SmallVector<
+ std::pair<std::function<void(__tgt_bin_desc *)>, __tgt_bin_desc *>>
+ DelayedBinDesc;
// List of all plugins, in use or not.
llvm::SmallVector<GenericPluginTy *> Plugins;
@@ -237,6 +245,10 @@ llvm::Error iteratePlatforms(CallbackTy Callback) {
"Failed to iterate platforms");
}
+/// Register \p Callback to serve the RPC calls of every active platform of this
+/// runtime's liboffload.
+void registerRPCCallback(unsigned (*Callback)(void *, unsigned));
+
// Iterate devices
template <typename CallbackTy>
llvm::Error iterateDevices(CallbackTy Callback, void *UserData) {
diff --git a/offload/include/Shared/Debug.h b/offload/include/Shared/Debug.h
index d6956fb75fdc2b..7e5470e23299a6 100644
--- a/offload/include/Shared/Debug.h
+++ b/offload/include/Shared/Debug.h
@@ -178,6 +178,7 @@ class LLVM_ABI odbg_ostream final : public raw_ostream {
uint32_t BaseLevel;
bool ShouldPrefixNextString;
bool ShouldEmitNewLineOnDestruction;
+ bool ShouldAbortOnDestruction;
bool NeedEndNewLine = false;
/// Buffer to reduce interference between different threads
@@ -229,17 +230,22 @@ class LLVM_ABI odbg_ostream final : public raw_ostream {
public:
explicit odbg_ostream(std::string Prefix, raw_ostream &Os, uint32_t BaseLevel,
bool ShouldPrefixNextString = true,
- bool ShouldEmitNewLineOnDestruction = true)
+ bool ShouldEmitNewLineOnDestruction = true,
+ bool ShouldAbortOnDestruction = false)
: Prefix(std::move(Prefix)), Os(Os), BaseLevel(BaseLevel),
ShouldPrefixNextString(ShouldPrefixNextString),
ShouldEmitNewLineOnDestruction(ShouldEmitNewLineOnDestruction),
- BufferStrm(Buffer) {
+ ShouldAbortOnDestruction(ShouldAbortOnDestruction), BufferStrm(Buffer) {
SetUnbuffered();
}
~odbg_ostream() final {
if (ShouldEmitNewLineOnDestruction && NeedEndNewLine)
BufferStrm << '\n';
Os << BufferStrm.str();
+ if (ShouldAbortOnDestruction) {
+ Os.flush();
+ abort();
+ }
}
odbg_ostream(const odbg_ostream &) = delete;
odbg_ostream &operator=(const odbg_ostream &) = delete;
@@ -248,6 +254,7 @@ class LLVM_ABI odbg_ostream final : public raw_ostream {
BaseLevel = other.BaseLevel;
ShouldPrefixNextString = other.ShouldPrefixNextString;
ShouldEmitNewLineOnDestruction = other.ShouldEmitNewLineOnDestruction;
+ ShouldAbortOnDestruction = other.ShouldAbortOnDestruction;
NeedEndNewLine = other.NeedEndNewLine;
Muted = other.Muted;
BufferStrm << other.BufferStrm.str();
@@ -617,7 +624,8 @@ constexpr const char *ODT_Tool = OLDT_Tool;
constexpr const char *ODT_Module = OLDT_Module;
constexpr const char *ODT_Interop = "Interop";
-static inline odbg_ostream reportErrorStream() {
+static inline odbg_ostream reportErrorStream(bool ShouldAbort,
+ std::string Prefix) {
#ifdef OMPTARGET_DEBUG
if (::llvm::offload::debug::isDebugEnabled()) {
uint32_t RealLevel = ODL_Error;
@@ -625,13 +633,26 @@ static inline odbg_ostream reportErrorStream() {
(ODT_Error), RealLevel))
return odbg_ostream{
::llvm::offload::debug::computePrefix(DEBUG_PREFIX, ODT_Error),
- ::llvm::offload::debug::dbgs(), RealLevel};
+ ::llvm::offload::debug::dbgs(),
+ RealLevel,
+ /*ShouldPrefixNextString=*/true,
+ /*ShouldEmitNewLineOnDestruction=*/true,
+ ShouldAbort};
else
- return odbg_ostream{"", ::llvm::nulls(), 1};
+ return odbg_ostream{"",
+ ::llvm::nulls(),
+ 1,
+ /*ShouldPrefixNextString=*/true,
+ /*ShouldEmitNewLineOnDestruction=*/true,
+ ShouldAbort};
}
#endif
- return odbg_ostream{GETNAME(TARGET_NAME) " error: ",
- ::llvm::offload::debug::dbgs(), ODL_Error};
+ return odbg_ostream{GETNAME(TARGET_NAME) + Prefix,
+ ::llvm::offload::debug::dbgs(),
+ ODL_Error,
+ /*ShouldPrefixNextString=*/true,
+ /*ShouldEmitNewLineOnDestruction=*/true,
+ ShouldAbort};
}
#ifdef OMPTARGET_DEBUG
@@ -699,8 +720,18 @@ static inline raw_ostream &operator<<(raw_ostream &Os, void *Ptr) {
#endif // OMPTARGET_DEBUG
+// New REPORT warning macro in the same style as ODBG
+#define REPORT_WARN() \
+ ::llvm::omp::target::debug::reportErrorStream(/*ShouldAbort=*/false, \
+ " warning: ")
+// New REPORT error macro in the same style as ODBG
+#define REPORT() \
+ ::llvm::omp::target::debug::reportErrorStream(/*ShouldAbort=*/false, \
+ " error: ")
// New REPORT macro in the same style as ODBG
-#define REPORT() ::llvm::omp::target::debug::reportErrorStream()
+#define REPORT_FATAL() \
+ ::llvm::omp::target::debug::reportErrorStream(/*ShouldAbort=*/true, \
+ " fatal error: ")
} // namespace llvm::omp::target::debug
diff --git a/offload/include/device.h b/offload/include/device.h
index 3af85f4e41186c..89188bbe730da7 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -186,6 +186,12 @@ struct DeviceTy {
// Data transfer. The transfer is enqueued on the queue associated with
// \p AsyncInfo; the *Sync variants below do not need a queue and always
// transfer the data synchronously.
+ int32_t
+ submitNonContigData(void *TgtPtrBegin, void *HstPtrBegin,
+ const NonContigDescTy &CopyInfo, AsyncInfoTy &AsyncInfo,
+ HostDataToTargetTy *Entry = nullptr,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
// Copy data from host to device
int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
AsyncInfoTy &AsyncInfo,
@@ -193,6 +199,12 @@ struct DeviceTy {
MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr,
const ident_t *Loc = nullptr, const char *Name = nullptr);
+ int32_t
+ retrieveNonContigData(void *HstPtrBegin, void *TgtPtrBegin,
+ const NonContigDescTy &CopyInfo, AsyncInfoTy &AsyncInfo,
+ HostDataToTargetTy *Entry = nullptr,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
// Copy data from device back to host
int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
AsyncInfoTy &AsyncInfo,
@@ -223,6 +235,11 @@ struct DeviceTy {
MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr,
const ident_t *Loc = nullptr);
+ /// Whether data can be exchanged directly with \p DstDevice.
+ bool isDataExchangable(const DeviceTy &DstDevice) const {
+ return RTL == DstDevice.RTL;
+ }
+
// Copy data from current device to destination device directly
int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
int64_t Size, AsyncInfoTy &AsyncInfo);
@@ -250,16 +267,29 @@ struct DeviceTy {
KernelReplayOutcomeTy *ReplayOutcome,
AsyncInfoTy &AsyncInfo);
+ // Enqueues a host function in the asynchronous queue.
+ int32_t enqueueHostCall(void (*Callback)(void *), void *UserData,
+ AsyncInfoTy &AsyncInfo);
+
/// Synchronize device/queue/event based on \p AsyncInfo and return
/// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
int32_t synchronize(AsyncInfoTy &AsyncInfo);
+ /// Synchronize device/queue/event based on \p AsyncInfo without releasing the
+ /// queue and return OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
+ int32_t synchronizeStatic(AsyncInfoTy &AsyncInfo);
+
/// Query for device/queue/event based completion on \p AsyncInfo in a
/// non-blocking manner and return OFFLOAD_SUCCESS/OFFLOAD_FAIL when
/// succeeds/fails. Must be called multiple times until AsyncInfo is
/// completed and AsyncInfo.isDone() returns true.
int32_t queryAsync(AsyncInfoTy &AsyncInfo);
+ /// Query the queue of \p AsyncInfo without releasing it and return 0 when all
+ /// its operations are done, 1 when they are pending and OFFLOAD_FAIL on
+ /// error.
+ int32_t queryAsyncStatic(AsyncInfoTy &AsyncInfo);
+
/// Event related interfaces.
/// {
/// Record the event based on status in AsyncInfo->Queue at the moment the
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index 01be2d9cae6b50..e4f9c8005ecb0a 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -26,7 +26,9 @@
#include <cstdint>
#include <deque>
#include <functional>
+#include <mutex>
#include <type_traits>
+#include <variant>
#include "llvm/ADT/SmallVector.h"
@@ -119,7 +121,8 @@ struct DeviceTy;
/// mistakes.
class AsyncInfoTy {
public:
- enum class SyncTy { BLOCKING, NON_BLOCKING };
+ enum class SyncTy { BLOCKING, NON_BLOCKING, STATIC_NON_BLOCKING };
+ using PostProcFuncTy = std::function<int()>;
private:
/// Locations we used in (potentially) asynchronous calls which should live
@@ -129,8 +132,8 @@ class AsyncInfoTy {
/// Post-processing operations executed after a successful synchronization.
/// \note the post-processing function should return OFFLOAD_SUCCESS or
/// OFFLOAD_FAIL appropriately.
- using PostProcFuncTy = std::function<int()>;
llvm::SmallVector<PostProcFuncTy> PostProcessingFunctions;
+ std::mutex PostProcessingFunctionsMutex;
ol_queue_handle_t Queue = nullptr;
DeviceTy &Device;
@@ -156,24 +159,37 @@ class AsyncInfoTy {
/// Get the underlying queue handle.
ol_queue_handle_t getQueue() const { return Queue; }
- /// Synchronize all pending actions.
+ /// Finalizes this instance of AsyncInfoTy.
///
- /// \note synchronization will be performance in a blocking or non-blocking
- /// manner, depending on the SyncType.
+ /// \note synchronization will be performed only if SyncType is blocking.
///
- /// \note if the operations are completed, the registered post-processing
- /// functions will be executed once and unregistered afterwards.
+ /// \note in all SyncType cases, if the operations are completed, the
+ /// registered post-processing functions will be executed once and
+ /// unregistered afterwards.
///
/// \returns OFFLOAD_FAIL or OFFLOAD_SUCCESS appropriately.
+ int finalize();
+
+ /// Synchronize all pending actions.
+ ///
+ /// \returns OFFLOAD_FAIL or OFFLOAD_SUCCESS depending on whether an error was
+ /// encountered.
int synchronize();
+ /// Queries whether all pending actions are done. This function does not
+ /// return the queue to the RTL.
+ ///
+ /// \returns OFFLOAD_FAIL on error, 0 when the actions are done, and 1 when
+ /// they are pending.
+ int query();
+
/// Return a void* reference with a lifetime that is at least as long as this
/// AsyncInfoTy object. The location can be used as intermediate buffer.
void *&getVoidPtrLocation();
/// Check if all asynchronous operations are completed.
///
- /// \note only a lightweight check. If needed, use synchronize() to query the
+ /// \note only a lightweight check. If needed, use finalize() to query the
/// status of AsyncInfo before checking.
///
/// \returns true if there is no pending asynchronous operations, false
@@ -189,6 +205,7 @@ class AsyncInfoTy {
static_assert(std::is_convertible_v<FuncTy, PostProcFuncTy>,
"Invalid post-processing function type. Please check "
"function signature!");
+ std::lock_guard<std::mutex> PPFGuard{PostProcessingFunctionsMutex};
PostProcessingFunctions.emplace_back(Function);
}
@@ -281,6 +298,48 @@ struct __tgt_target_non_contig {
uint64_t Stride;
};
+struct NonContigDescTy {
+ llvm::SmallVector<__tgt_target_non_contig, 6> Dims;
+
+ const __tgt_target_non_contig &getDim(unsigned I) { return Dims[I]; }
+ unsigned getRank() const { return Dims.size(); }
+
+ /// Drop the innermost dimensions that are already covered by the contiguous
+ /// chunk the copy moves for the final dimension. A dimension is absorbed only
+ /// when it exactly tiles the stride of the one enclosing it, and only while
+ /// every dimension inside it was absorbed too: dropping a dimension that
+ /// still has gaps would widen the chunk over elements outside the region.
+ void mergeContiguousDims() {
+ int RemovedDim = 0;
+ for (int I = getRank() - 1; I > 0; --I) {
+ if (Dims[I].Count * Dims[I].Stride != Dims[I - 1].Stride)
+ break;
+ RemovedDim++;
+ }
+ Dims.resize(getRank() - RemovedDim);
+ }
+
+ uint64_t getLastDimCopySize() const {
+ return Dims.back().Count * Dims.back().Stride;
+ }
+
+ /// Bytes spanned from the start of the region to the end of its last chunk.
+ /// The copy walks the final dimension as one contiguous chunk while the
+ /// preceding dimensions only pick where those chunks start, so each of them
+ /// reaches its last starting position rather than a whole stride beyond it;
+ /// the gap after that final chunk belongs to whatever surrounds the region.
+ uint64_t getAllocSize() const {
+ for (const auto &Dim : Dims)
+ if (Dim.Count == 0)
+ return 0;
+
+ uint64_t Size = getLastDimCopySize();
+ for (unsigned I = 0; I + 1 < getRank(); ++I)
+ Size += (Dims[I].Count - 1 + Dims[I].Offset) * Dims[I].Stride;
+ return Size;
+ }
+};
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -370,6 +429,12 @@ void __tgt_target_data_begin_mapper(ident_t *Loc, int64_t DeviceId,
void **Args, int64_t *ArgSizes,
int64_t *ArgTypes, map_var_info_t *ArgNames,
void **ArgMappers);
+void __tgt_target_data_begin_async_mapper(ident_t *Loc, int64_t DeviceId,
+ int32_t ArgNum, void **ArgsBase,
+ void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes,
+ map_var_info_t *ArgNames,
+ void **ArgMappers, int32_t Queue);
void __tgt_target_data_begin_nowait_mapper(
ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,
@@ -390,6 +455,12 @@ void __tgt_target_data_end_mapper(ident_t *Loc, int64_t DeviceId,
int32_t ArgNum, void **ArgsBase, void **Args,
int64_t *ArgSizes, int64_t *ArgTypes,
map_var_info_t *ArgNames, void **ArgMappers);
+void __tgt_target_data_end_async_mapper(ident_t *Loc, int64_t DeviceId,
+ int32_t ArgNum, void **ArgsBase,
+ void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes,
+ map_var_info_t *ArgNames,
+ void **ArgMappers, int32_t Queue);
void __tgt_target_data_end_nowait_mapper(
ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,
@@ -417,6 +488,12 @@ void __tgt_target_data_update_nowait_mapper(
void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,
void **ArgMappers, int32_t DepNum, void *DepList, int32_t NoAliasDepNum,
void *NoAliasDepList);
+void __tgt_target_data_update_async_mapper(ident_t *Loc, int64_t DeviceId,
+ int32_t ArgNum, void **ArgsBase,
+ void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes,
+ map_var_info_t *ArgNames,
+ void **ArgMappers, int32_t Queue);
// Performs the same actions as data_begin in case ArgNum is non-zero
// and initiates run of offloaded region on target platform; if ArgNum
@@ -427,6 +504,10 @@ void __tgt_target_data_update_nowait_mapper(
int __tgt_target_kernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
int32_t ThreadLimit, void *HostPtr, KernelArgsTy *Args);
+int __tgt_target_kernel_async(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
+ int32_t ThreadLimit, void *HostPtr,
+ KernelArgsTy *KernelArgs, int32_t Queue);
+
// Non-blocking synchronization for target nowait regions. This function
// acquires the asynchronous context from task data of the current task being
// executed and tries to query for the completion of its operations. If the
diff --git a/offload/libacctarget/CMakeLists.txt b/offload/libacctarget/CMakeLists.txt
new file mode 100644
index 00000000000000..60b61269f332bd
--- /dev/null
+++ b/offload/libacctarget/CMakeLists.txt
@@ -0,0 +1,71 @@
+message(STATUS "Building OpenACC offloading runtime library libacctarget.")
+
+remove_definitions(-DOMPT_SUPPORT=1)
+
+add_library(acctarget SHARED
+ Interface.cpp
+ RuntimeState.cpp
+
+ DeviceManager.cpp
+ QueueManager.cpp
+
+ RuntimeImpl.cpp
+ RuntimeInterface.cpp
+ DataRuntimeInterface.cpp
+ CfiDataRuntimeInterface.cpp
+
+ ${LIBOMPACCSUPPORT_SOURCES}
+)
+
+if(LLVM_LINK_LLVM_DYLIB)
+ set(llvm_libs LLVM)
+else()
+ llvm_map_components_to_libnames(llvm_libs FrontendOpenMP Support)
+endif()
+target_link_libraries(acctarget PRIVATE ${llvm_libs})
+target_include_directories(acctarget PRIVATE
+ ${LIBOMPTARGET_INCLUDE_DIR} ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include
+ ${CMAKE_CURRENT_BINARY_DIR}/../plugins-nextgen/common/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/include
+)
+
+if(LLVM_HAVE_LINK_VERSION_SCRIPT)
+ target_link_libraries(acctarget PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+ set_property(TARGET acctarget APPEND PROPERTY LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/exports)
+endif()
+
+# Define the TARGET_NAME and DEBUG_PREFIX.
+target_compile_definitions(acctarget PRIVATE
+ TARGET_NAME=acctarget
+ DEBUG_PREFIX="acctarget"
+)
+
+
+target_link_libraries(acctarget PRIVATE LLVMOffloadStatic)
+
+# Link against flang_rt.runtime for Fortran descriptor support.
+# flang_rt.runtime is a sibling runtime; link against the shared library target.
+if(TARGET flang_rt.runtime.dynamic)
+ target_link_libraries(acctarget PRIVATE flang_rt.runtime.dynamic)
+elseif(TARGET flang_rt.runtime.static)
+ target_link_libraries(acctarget PRIVATE flang_rt.runtime.static)
+else()
+ message(FATAL_ERROR "flang_rt.runtime target not found")
+endif()
+
+target_compile_options(acctarget PRIVATE ${offload_compile_flags})
+target_link_options(acctarget PRIVATE ${offload_link_flags})
+
+# libacctarget.so needs to be aware of where the plugins live as they
+# are now separated in the build directory.
+set_target_properties(acctarget PROPERTIES
+ POSITION_INDEPENDENT_CODE ON
+ INSTALL_RPATH "${OFFLOAD_INSTALL_RPATH}"
+ BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+if(UNIX AND NOT APPLE)
+ set_target_properties(acctarget PROPERTIES
+ SOVERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}${LLVM_VERSION_SUFFIX}"
+ VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}${LLVM_VERSION_SUFFIX}")
+endif()
+install(TARGETS acctarget LIBRARY COMPONENT offload DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
diff --git a/offload/libacctarget/CfiDataRuntimeInterface.cpp b/offload/libacctarget/CfiDataRuntimeInterface.cpp
new file mode 100644
index 00000000000000..aa17014117d363
--- /dev/null
+++ b/offload/libacctarget/CfiDataRuntimeInterface.cpp
@@ -0,0 +1,297 @@
+//===- CfiAccEntryImpl.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DeviceManager.h"
+#include "Logger.h"
+#include "Private.h"
+#include "omptarget.h"
+
+#include "Interface.h"
+#include "include/openacc.h"
+
+using namespace llvm::acc::target;
+using namespace llvm::acc::target::debug;
+
+#define PREAMBLE() \
+ FUNC_LOGGER(); \
+ AccDataDescF18 AccDesc{{TGT_ACC_DESC_F18}, &Desc->raw()}; \
+ AccDataDescF18 *AccDataDescs[] = {&AccDesc}; \
+ void *ArgPtrs[] = {reinterpret_cast<void *>(&Desc->raw())}; \
+ void *ArgBasePtrs[] = {nullptr}; \
+ int64_t ArgSizes[] = {0}; \
+ int64_t ArgTypes[] = {TGT_ACC_MAPTYPE_NONE};
+
+extern "C" {
+int _cfi_acc_is_present_a(const Fortran::runtime::Descriptor *Desc) {
+ return accIsPresent(Desc->OffsetElement());
+}
+
+int _cfi_acc_create_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_create_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+int _cfi_acc_pcreate_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_pcreate_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+int _cfi_acc_present_or_create_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_present_or_create_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_delete_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_delete_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+int _cfi_acc_delete_finalize_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_delete_finalize_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_copyin_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_copyin_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+int _cfi_acc_pcopyin_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_pcopyin_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+int _cfi_acc_present_or_copyin_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_present_or_copyin_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_enter(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_copyout_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_copyout_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_copyout_finalize_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM | TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_copyout_finalize_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM | TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_update_device_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_update_device_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_updatein_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_updatein_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_update_self_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_update_self_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_update_host_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_update_host_async_a(Fortran::runtime::Descriptor *Desc,
+ int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+
+int _cfi_acc_updateout_a(Fortran::runtime::Descriptor *Desc) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+ return 0;
+}
+int _cfi_acc_updateout_async_a(Fortran::runtime::Descriptor *Desc, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+ return 0;
+}
+}
diff --git a/offload/libacctarget/DataRuntimeInterface.cpp b/offload/libacctarget/DataRuntimeInterface.cpp
new file mode 100644
index 00000000000000..e6111932e8e77e
--- /dev/null
+++ b/offload/libacctarget/DataRuntimeInterface.cpp
@@ -0,0 +1,204 @@
+//===- AccEntryImpl.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Logger.h"
+#include "Private.h"
+
+#include "Interface.h"
+#include "include/openacc.h"
+
+using namespace llvm::acc::target;
+using namespace llvm::acc::target::debug;
+
+#define PREAMBLE() \
+ FUNC_LOGGER(); \
+ AccDataDescF18 *AccDataDescs[] = {nullptr}; \
+ void *ArgPtrs[] = {Ptr}; \
+ void *ArgBasePtrs[] = {nullptr}; \
+ int64_t ArgSizes[] = {static_cast<int64_t>(Bytes)}; \
+ int64_t ArgTypes[] = {TGT_ACC_MAPTYPE_NONE};
+
+extern "C" {
+int acc_is_present(void *Ptr) { return accIsPresent(Ptr); }
+
+void *acc_create(void *Ptr, size_t Bytes) {
+ return accDataEnter(nullptr, Ptr, Bytes, TGT_ACC_MAPTYPE_NONE,
+ acc_async_sync);
+}
+void acc_create_async(void *Ptr, size_t Bytes, int Async) {
+ accDataEnter(nullptr, Ptr, Bytes, TGT_ACC_MAPTYPE_NONE, acc_async_sync);
+}
+void *acc_pcreate(void *Ptr, size_t Bytes) { return acc_create(Ptr, Bytes); }
+void acc_pcreate_async(void *Ptr, size_t Bytes, int Async) {
+ acc_create_async(Ptr, Bytes, Async);
+}
+void *acc_present_or_create(void *Ptr, size_t Bytes) {
+ return acc_create(Ptr, Bytes);
+}
+void acc_present_or_create_async(void *Ptr, size_t Bytes, int Async) {
+ acc_create_async(Ptr, Bytes, Async);
+}
+
+void acc_delete(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_delete_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+void acc_delete_finalize(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_delete_finalize_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void *acc_copyin(void *Ptr, size_t Bytes) {
+ return accDataEnter(nullptr, Ptr, Bytes, TGT_ACC_MAPTYPE_TO, acc_async_sync);
+}
+void acc_copyin_async(void *Ptr, size_t Bytes, int Async) {
+ accDataEnter(nullptr, Ptr, Bytes, TGT_ACC_MAPTYPE_TO, acc_async_sync);
+}
+void *acc_pcopyin(void *Ptr, size_t Bytes) { return acc_copyin(Ptr, Bytes); }
+void acc_pcopyin_async(void *Ptr, size_t Bytes, int Async) {
+ acc_copyin_async(Ptr, Bytes, Async);
+}
+void *acc_present_or_copyin(void *Ptr, size_t Bytes) {
+ return acc_copyin(Ptr, Bytes);
+}
+void acc_present_or_copyin_async(void *Ptr, size_t Bytes, int Async) {
+ acc_copyin_async(Ptr, Bytes, Async);
+}
+
+void acc_copyout(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_copyout_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void acc_copyout_finalize(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM | TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_copyout_finalize_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM | TGT_ACC_MAPTYPE_FINALIZE;
+ __tgt_acc_data_exit(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void acc_update_device(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_update_device_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void acc_updatein(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_updatein_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_TO;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void acc_update_self(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_update_self_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void acc_update_host(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_update_host_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+
+void acc_updateout(void *Ptr, size_t Bytes) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs),
+ acc_async_sync);
+}
+void acc_updateout_async(void *Ptr, size_t Bytes, int Async) {
+ PREAMBLE();
+ ArgTypes[0] = TGT_ACC_MAPTYPE_FROM;
+ __tgt_acc_data_update(nullptr, 0, acc_device_default, 1, ArgBasePtrs, ArgPtrs,
+ ArgSizes, ArgTypes, nullptr, nullptr,
+ reinterpret_cast<AccDataDesc **>(AccDataDescs), Async);
+}
+}
diff --git a/offload/libacctarget/Debug.h b/offload/libacctarget/Debug.h
new file mode 100644
index 00000000000000..21624c42a1ab06
--- /dev/null
+++ b/offload/libacctarget/Debug.h
@@ -0,0 +1,26 @@
+//===- Debug.h --------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBACCTARGET_DEBUG_H_
+#define _LIBACCTARGET_DEBUG_H_
+
+#include "Shared/Debug.h"
+
+namespace llvm::acc::target::debug {
+
+// Debug types to use in libacctarget
+constexpr const char *ADT_Init = "ACCInit";
+constexpr const char *ADT_Mapping = "ACCMapping";
+constexpr const char *ADT_Descriptor = "ACCDescriptor";
+constexpr const char *ADT_Queue = "ACCQueue";
+constexpr const char *ADT_Interface = "ACCInterface";
+constexpr const char *ADT_Kernel = "ACCKernel";
+
+} // namespace llvm::acc::target::debug
+
+#endif // _LIBACCTARGET_DEBUG_H_
diff --git a/offload/libacctarget/DeviceManager.cpp b/offload/libacctarget/DeviceManager.cpp
new file mode 100644
index 00000000000000..eb6bc1d824a1a8
--- /dev/null
+++ b/offload/libacctarget/DeviceManager.cpp
@@ -0,0 +1,232 @@
+//===- DeviceManager.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DeviceManager.h"
+#include "PluginManager.h"
+#include "RuntimeState.h"
+#include "openacc.h"
+
+// OpenACC 3.4, sec. 2.3.1 "Modifying and Retrieving ICV Values"
+// Each host thread needs its own value, thus these are `thread_local`.
+//
+// The DeviceManager owns these ICVs and they should not be accessible outside
+// and are thus static.
+namespace llvm::acc::target::icv {
+/// OpenACC 3.4, sec. 2.3 "Internal Control Variables"
+/// "acc-current-device-num-var - controls which device of the selected type is
+/// used."
+/// TODO can we use PerThreadTable here?
+static thread_local std::array<DeviceManagerTy::DeviceIdTy,
+ AccDeviceNumConcreteTypes>
+ AccCurrentDeviceNumVar = {0};
+/// OpenACC 3.4, sec. 2.3 "Internal Control Variables"
+/// "acc-current-device-type-var - controls which type of device is used."
+static thread_local acc_device_t AccCurrentDeviceTypeVar = acc_device_default;
+/// The device type to use when the default is asked for. Initially we set it to
+/// none. When the plugins get initialized we will set the default to one of the
+/// target device types we have available.
+static acc_device_t AccCurrentDefaultDeviceTypeVar = acc_device_none;
+
+} // namespace llvm::acc::target::icv
+
+namespace llvm::acc::target {
+DeviceManagerTy *DM = nullptr;
+} // namespace llvm::acc::target
+
+using namespace llvm::acc::target;
+
+static const char *accDeviceToStr(acc_device_t DeviceType) {
+ switch (DeviceType) {
+ case acc_device_nvidia:
+ return "nvidia";
+ case acc_device_amd:
+ return "amd";
+ case acc_device_spirv:
+ return "spirv";
+ case acc_device_none:
+ return "<none>";
+ case acc_device_default:
+ return "<default>";
+ case acc_device_host:
+ return "<host>";
+ case acc_device_not_host:
+ return "<not_host>";
+ default:
+ return "<unknown>";
+ }
+}
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+ acc_device_t DeviceType) {
+ return OS << accDeviceToStr(DeviceType) << " ("
+ << static_cast<int>(DeviceType) << ")";
+}
+
+DeviceManagerTy::SingleDeviceTypeMapTy &
+DeviceManagerTy::getSingleDeviceTypeMap(acc_device_t DeviceType) {
+ return PMDeviceMap[DeviceType - AccDeviceTypeOffset];
+}
+
+void DeviceManagerTy::init() {
+ refreshDeviceMapping(/*UpdateDeviceType=*/true);
+}
+
+void DeviceManagerTy::deinit() {}
+
+void DeviceManagerTy::refreshDeviceMapping(bool UpdateDeviceType) {
+ assert(this);
+
+ for (int DeviceTypeInt = acc_device_concrete_type_begin;
+ DeviceTypeInt < acc_device_concrete_type_end; DeviceTypeInt++)
+ getSingleDeviceTypeMap(static_cast<acc_device_t>(DeviceTypeInt)).resize(0);
+
+ auto ExclusiveDevicesAccessor = PM->getExclusiveDevicesAccessor();
+ for (DeviceTy &Device : PM->devices(ExclusiveDevicesAccessor)) {
+ if (Device.RTL->getTripleArch() == llvm::Triple::nvptx64) {
+ getSingleDeviceTypeMap(acc_device_nvidia).push_back(Device.DeviceID);
+ } else if (Device.RTL->getTripleArch() == llvm::Triple::amdgpu) {
+ getSingleDeviceTypeMap(acc_device_amd).push_back(Device.DeviceID);
+ } else if (Device.RTL->getTripleArch() == llvm::Triple::spirv64) {
+ getSingleDeviceTypeMap(acc_device_spirv).push_back(Device.DeviceID);
+ }
+ }
+
+ ODBG() << "Refreshed OpenACC devices:";
+ for (int DeviceTypeInt = acc_device_concrete_type_begin;
+ DeviceTypeInt < acc_device_concrete_type_end; DeviceTypeInt++) {
+ acc_device_t DeviceType = static_cast<acc_device_t>(DeviceTypeInt);
+ unsigned Num = getSingleDeviceTypeMap(DeviceType).size();
+ ODBG() << " Type " << DeviceType;
+ for (unsigned I = 0; I < Num; I++) {
+ ODBG() << " OpenACC Device #" << I << " -> PM Device #"
+ << getSingleDeviceTypeMap(DeviceType)[I];
+ }
+ }
+
+ if (UpdateDeviceType) {
+ // Set the default current device type to a device we have available in the
+ // below order of priority.
+ auto CheckType = [&](acc_device_t Type) {
+ if (getSingleDeviceTypeMap(Type).size() > 0) {
+ assert(Type >= acc_device_concrete_type_begin &&
+ Type < acc_device_concrete_type_end &&
+ "We should only set AccCurrentDefaultDeviceTypeVar to a "
+ "concrete type");
+ ODBG() << "Updating AccCurrentDefaultDeviceTypeVar to " << Type;
+ icv::AccCurrentDefaultDeviceTypeVar = Type;
+ return true;
+ }
+ return false;
+ };
+ false || CheckType(acc_device_nvidia) || CheckType(acc_device_amd) ||
+ CheckType(acc_device_spirv) || CheckType(acc_device_host);
+ }
+}
+
+int DeviceManagerTy::getPMDeviceId(acc_device_t DeviceType) {
+ ODBG() << "Getting device for " << DeviceType;
+ if (DeviceType == acc_device_none) {
+ DeviceType = icv::AccCurrentDeviceTypeVar;
+ ODBG() << "Correcting to current type " << DeviceType;
+ }
+ if (DeviceType == acc_device_default) {
+ ODBG() << "Corrected to value of AccCurrentDefaultDeviceTypeVar: "
+ << icv::AccCurrentDefaultDeviceTypeVar;
+ DeviceType = icv::AccCurrentDefaultDeviceTypeVar;
+ }
+ ODBG() << "Current device has id " << icv::AccCurrentDeviceNumVar[DeviceType];
+ checkICVs();
+ return getSingleDeviceTypeMap(
+ DeviceType)[icv::AccCurrentDeviceNumVar[DeviceType]];
+}
+
+int DeviceManagerTy::getPMDeviceId() {
+ ODBG() << "Getting current device, type " << icv::AccCurrentDeviceTypeVar;
+ checkICVs();
+ return getPMDeviceId(icv::AccCurrentDeviceTypeVar);
+}
+
+int DeviceManagerTy::getDeviceId(acc_device_t DeviceType) {
+ checkICVs();
+ return icv::AccCurrentDeviceNumVar[DeviceType];
+}
+
+void DeviceManagerTy::checkICVs() {
+ ODBG() << "acc-current-device-type = " << icv::AccCurrentDeviceTypeVar;
+ for (int DeviceTypeInt = acc_device_concrete_type_begin;
+ DeviceTypeInt < acc_device_concrete_type_end; DeviceTypeInt++) {
+ acc_device_t DeviceType = static_cast<acc_device_t>(DeviceTypeInt);
+ ODBG() << "acc-current-device-num[" << DeviceType
+ << "] = " << icv::AccCurrentDeviceNumVar[DeviceType];
+ }
+ ODBG() << "acc-current-device-type = " << icv::AccCurrentDeviceTypeVar;
+ assert(icv::AccCurrentDeviceTypeVar == acc_device_default ||
+ (icv::AccCurrentDeviceTypeVar >= acc_device_concrete_type_begin &&
+ icv::AccCurrentDeviceTypeVar < acc_device_concrete_type_end));
+ acc_device_t DeviceType = icv::AccCurrentDeviceTypeVar;
+ if (DeviceType == acc_device_default) {
+ DeviceType = icv::AccCurrentDefaultDeviceTypeVar;
+ ODBG() << "Corrected to value of AccCurrentDefaultDeviceTypeVar: "
+ << icv::AccCurrentDefaultDeviceTypeVar;
+ }
+ ODBG() << icv::AccCurrentDeviceNumVar[DeviceType];
+ assert(icv::AccCurrentDeviceNumVar[DeviceType] <
+ static_cast<int64_t>(getSingleDeviceTypeMap(DeviceType).size()));
+}
+
+int DeviceManagerTy::getNumDevices(acc_device_t DeviceType) {
+ checkICVs();
+ return getSingleDeviceTypeMap(DeviceType).size();
+}
+
+void DeviceManagerTy::setAllDeviceId(int DevNum) {
+ for (auto &CurrDevNum : icv::AccCurrentDeviceNumVar) {
+ CurrDevNum = DevNum;
+ }
+ checkICVs();
+}
+
+void DeviceManagerTy::setDeviceId(acc_device_t DeviceType, int DevNum) {
+ icv::AccCurrentDeviceNumVar[DeviceType] = DevNum;
+ checkICVs();
+}
+
+void DeviceManagerTy::setDeviceId(int DevNum) {
+ setDeviceId(icv::AccCurrentDeviceTypeVar, DevNum);
+ checkICVs();
+}
+
+acc_device_t DeviceManagerTy::getDeviceType() {
+ checkICVs();
+ return icv::AccCurrentDeviceTypeVar;
+}
+
+void DeviceManagerTy::setDeviceType(acc_device_t DeviceType) {
+ icv::AccCurrentDeviceTypeVar = DeviceType;
+ checkICVs();
+}
+
+size_t DeviceManagerTy::getDeviceProperty(int, acc_device_t,
+ acc_device_property_t) {
+ REPORT_FATAL() << "device properties not yet implemented";
+ return 0;
+}
+
+const char *DeviceManagerTy::getDevicePropertyString(int, acc_device_t,
+ acc_device_property_t) {
+ REPORT_FATAL() << "device properties not yet implemented";
+ return "";
+}
+
+llvm::Expected<DeviceTy &> DeviceManagerTy::getDevice(acc_device_t DeviceType) {
+ return PM->getDevice(getPMDeviceId(DeviceType));
+}
+
+llvm::Expected<DeviceTy &> DeviceManagerTy::getDevice() {
+ return PM->getDevice(getPMDeviceId());
+}
diff --git a/offload/libacctarget/DeviceManager.h b/offload/libacctarget/DeviceManager.h
new file mode 100644
index 00000000000000..d825eeb1c21609
--- /dev/null
+++ b/offload/libacctarget/DeviceManager.h
@@ -0,0 +1,75 @@
+//===- DeviceManager.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ACC_OFFLOAD_DEVICE_MANAGER_H_
+#define LLVM_ACC_OFFLOAD_DEVICE_MANAGER_H_
+
+#include "include/openacc.h"
+#include "omptarget.h"
+#include <array>
+#include <cstddef>
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+ acc_device_t DeviceType);
+
+namespace llvm::acc::target {
+
+constexpr int AccDeviceNumConcreteTypes =
+ acc_device_concrete_type_end - acc_device_concrete_type_begin;
+constexpr int AccDeviceTypeOffset = acc_device_concrete_type_begin;
+
+class DeviceManagerTy {
+public:
+ using DeviceIdTy = int64_t;
+
+private:
+ using SingleDeviceTypeMapTy = llvm::SmallVector<DeviceIdTy, 8>;
+ using AllDeviceTypeMap =
+ std::array<SingleDeviceTypeMapTy, AccDeviceNumConcreteTypes>;
+ AllDeviceTypeMap PMDeviceMap;
+
+ SingleDeviceTypeMapTy &getSingleDeviceTypeMap(acc_device_t DeviceType);
+
+public:
+ void init();
+ void deinit();
+
+ // Refreshes the device mapping according to the
+ void refreshDeviceMapping(bool UpdateDeviceType);
+
+ // Entry points for ACC APIs.
+ int getDeviceId(acc_device_t DeviceType);
+ int getNumDevices(acc_device_t DeviceType);
+
+ void setAllDeviceId(int DeviceId);
+ void setDeviceId(acc_device_t DeviceType, int DeviceId);
+ void setDeviceId(int DeviceId);
+
+ acc_device_t getDeviceType();
+ void setDeviceType(acc_device_t DeviceType);
+
+ size_t getDeviceProperty(int DeviceId, acc_device_t DeviceType,
+ acc_device_property_t DeviceProperty);
+ const char *getDevicePropertyString(int DeviceId, acc_device_t DeviceType,
+ acc_device_property_t DeviceProperty);
+
+ // Verification.
+ void checkICVs();
+
+ // Obtaining the device ID for use with PluginManager.
+ int getPMDeviceId(acc_device_t DeviceType);
+ int getPMDeviceId();
+
+ llvm::Expected<DeviceTy &> getDevice(acc_device_t DeviceType);
+ llvm::Expected<DeviceTy &> getDevice();
+};
+
+extern DeviceManagerTy *DM;
+} // namespace llvm::acc::target
+
+#endif // LLVM_ACC_OFFLOAD_DEVICE_MANAGER_H_
diff --git a/offload/libacctarget/Interface.cpp b/offload/libacctarget/Interface.cpp
new file mode 100644
index 00000000000000..02a10e977101d1
--- /dev/null
+++ b/offload/libacctarget/Interface.cpp
@@ -0,0 +1,2427 @@
+//===- Interface.cpp --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Interface.h"
+#include "Debug.h"
+#include "DeviceManager.h"
+#include "Logger.h"
+#include "OpenMP/Mapping.h"
+#include "PluginManager.h"
+#include "Private.h"
+#include "QueueManager.h"
+#include "RuntimeState.h"
+#include "Shared/APITypes.h"
+#include "Shared/Debug.h"
+#include "Shared/SourceInfo.h"
+#include "device.h"
+#include "omptarget.h"
+#include "openacc.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdlib>
+#include <iostream>
+#include <limits>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <optional>
+#include <sstream>
+#include <string.h>
+#include <string>
+#include <type_traits>
+#include <variant>
+#include <vector>
+
+using namespace llvm::acc::target;
+using namespace llvm::acc::target::debug;
+
+using llvm::SmallVector;
+
+namespace {
+constexpr int32_t AccAsyncSync = acc_async_sync;
+constexpr int32_t AccAsyncDefault = acc_async_default;
+constexpr int32_t AccAsyncNoval = acc_async_noval;
+constexpr int32_t AccAsyncDefaultQueue = -5;
+} // namespace
+
+namespace llvm::acc::target::icv {
+// acc-default-async-var
+thread_local int32_t AccDefaultAsyncVar = AccAsyncDefaultQueue;
+} // namespace llvm::acc::target::icv
+
+namespace {
+// TODO hook up to some env var
+bool Pedantic = true;
+
+#define STR_AND_VAL(X) #X << " " << X
+#define STR_AND_VALI(X) #X << " " << (int64_t)X
+#define OPT_STR_AND_VAL(X) #X << " " << (X ? std::to_string(*X) : "(nil)")
+#define SOPT_STR_AND_VAL(X) #X << " " << (X ? X : "(nil)")
+
+struct DescMappingInfoTy {
+ // The size of the descriptor
+ size_t DescriptorSize = 0;
+ // The offset in the host descriptor where the pointer to the raw memory is
+ // stored.
+ size_t RawMemoryPtrOffset = 0;
+};
+
+struct MemMappingInfoTy {
+ void *RawMemoryPtr = nullptr;
+ void *RawMemoryBasePtr = nullptr;
+ std::optional<uint64_t> RawMemorySize = std::nullopt;
+ std::optional<NonContigDescTy> CopyDesc = std::nullopt;
+
+ ptrdiff_t getBaseDelta() {
+ ptrdiff_t Delta = reinterpret_cast<intptr_t>(RawMemoryPtr) -
+ reinterpret_cast<intptr_t>(RawMemoryBasePtr);
+ return Delta;
+ }
+ void verify() { assert(RawMemoryPtr); }
+ void dump(llvm::raw_ostream &OS) {
+ OS << "MemMappingInfoTy:\n";
+ // clang-format off
+ OS
+ << " " << STR_AND_VAL(RawMemoryPtr)
+ << " " << STR_AND_VAL(RawMemoryBasePtr)
+ << " " << OPT_STR_AND_VAL(RawMemorySize)
+ << " " << !!CopyDesc
+ << "\n";
+ // clang-format on
+ }
+};
+
+struct AccArrayDim {
+ long Offset;
+ long Stride;
+ long Size;
+ long Extent;
+};
+
+struct ArrayInfo {
+ std::vector<AccArrayDim> Dims;
+ // The size of the raw memory allocation.
+ std::optional<uint64_t> RawMemorySize = {};
+ // The address of the host memory to be copied.
+ void *RawMemoryAddr = nullptr;
+ // Size of the array element.
+ int64_t ElementSize = 0;
+
+ void setPtr(void *Ptr) { RawMemoryAddr = Ptr; }
+
+ std::optional<int64_t> getElementCountInDim(unsigned I) {
+ // Prefer the `size` instead of `extent`. This is due to cases like this:
+ //
+ // real a0(100)
+ // call acc_copyin(a0(1:99))
+ // !$acc present(a0(1:99))
+ //
+ // Where the `acc_copyin` will allocate space for 99 elements because we
+ // parse the flang descriptor which only contains information on the size
+ // (99), but then if we use the `extent` from the `acc present`, we would
+ // require 100 elements, which is larger than the previously allocated
+ // memory. Thus, we use the `size`.
+
+ if (Dims[I].Size != -1) {
+ return Dims[I].Size;
+ }
+ if (Dims[I].Extent != -1) {
+ return Dims[I].Extent;
+ }
+
+ return std::nullopt;
+ }
+
+ void computeSizeFromDims(ident_t *Loc) {
+ if (Dims.size() == 0) {
+ RawMemorySize = ElementSize;
+ return;
+ }
+
+ // The memory spans from the first element to the end of the last one. A
+ // dimension whose stride exceeds the element size leaves a gap after its
+ // final element, and that gap belongs to whatever contains this array, so
+ // charging a full stride for the last element would claim memory the array
+ // does not occupy.
+ uint64_t Span = ElementSize;
+ for (unsigned I = 0; I < Dims.size(); I++) {
+ if (Dims[I].Stride < 0) {
+ REPORT_FATAL() << Loc << "Unsupported negative stride";
+ }
+
+ std::optional<int64_t> Count = getElementCountInDim(I);
+ if (!Count) {
+ RawMemorySize = std::nullopt;
+ ODBG(ADT_Descriptor) << "Unknown element count in dimension " << I
+ << ", cannot compute RawMemorySize";
+ return;
+ }
+ if (*Count == 0) {
+ RawMemorySize = 0;
+ ODBG(ADT_Descriptor) << "Empty dimension " << I;
+ return;
+ }
+
+ Span += (*Count - 1) * Dims[I].Stride * ElementSize;
+ }
+
+ RawMemorySize = Span;
+ ODBG(ADT_Descriptor) << "Computed " << OPT_STR_AND_VAL(RawMemorySize);
+ }
+
+ void normalize() {
+ normalizeStrides();
+ normalizeOffsets();
+ }
+
+ bool hasNormalizedStrides() {
+ for (std::size_t i = 0; i < Dims.size(); i++) {
+ if (Dims[i].Stride < 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void normalizeStrides() {
+ FUNC_LOGGER();
+ ODBG_IF([&]() { dump(llvm::dbgs()); });
+
+ if (hasNormalizedStrides()) {
+ ODBG(ADT_Descriptor) << "No normalization needed.";
+ return;
+ }
+ ODBG(ADT_Descriptor) << "Descriptor needs normalization.";
+
+ // The runtime cannot map negative stride arrays. So we must find the base
+ // address of the host pointer and then invert the descriptor so that the
+ // strides in all dimensions are positive. The base pointer delta will be
+ // used to attach the adjusted device pointer to the array descriptor - that
+ // is, the F18 descriptor will contain the end address of the array because
+ // that is what the compiler assumes.
+ int64_t baseHostPtrDeltaInBytes = 0;
+ for (std::size_t i = 0; i < Dims.size(); i++) {
+ if (Dims[i].Stride < 0) {
+ Dims[i].Stride = -Dims[i].Stride;
+ Dims[i].Offset = Dims[i].Extent - Dims[i].Size - Dims[i].Offset;
+
+ // For each negative stride, skip to previously accounted array.
+ baseHostPtrDeltaInBytes += Dims[i].Stride * (Dims[i].Extent - 1);
+ }
+ }
+
+ baseHostPtrDeltaInBytes *= ElementSize;
+
+ RawMemoryAddr =
+ reinterpret_cast<char *>(RawMemoryAddr) - baseHostPtrDeltaInBytes;
+
+ ODBG(ADT_Descriptor) << "Normalized:";
+ ODBG_IF([&]() { dump(llvm::dbgs()); });
+ }
+
+ bool hasNormalizedOffsets() {
+ for (std::size_t i = 0; i < Dims.size(); i++) {
+ if (Dims[i].Offset != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void normalizeOffsets() {
+ FUNC_LOGGER();
+ ODBG_IF([&]() { dump(llvm::dbgs()); });
+ assert(hasNormalizedStrides());
+
+ if (hasNormalizedOffsets()) {
+ ODBG(ADT_Descriptor) << "No normalization needed.";
+ return;
+ }
+ ODBG(ADT_Descriptor) << "Descriptor needs normalization.";
+
+ int64_t baseHostPtrDeltaInBytes = 0;
+ for (auto &Dim : Dims) {
+ if (Dim.Offset != 0) {
+ baseHostPtrDeltaInBytes += Dim.Offset * Dim.Stride;
+ Dim.Offset = 0;
+ }
+ }
+
+ baseHostPtrDeltaInBytes *= ElementSize;
+
+ RawMemoryAddr =
+ reinterpret_cast<char *>(RawMemoryAddr) + baseHostPtrDeltaInBytes;
+
+ ODBG(ADT_Descriptor) << "Normalized:";
+ ODBG_IF([&]() { dump(llvm::dbgs()); });
+ }
+
+ void verify() {
+ assert(ElementSize > 0);
+ for (unsigned I = 0; I < Dims.size() - 1; I++) {
+ assert(Dims[I].Stride < Dims[I + 1].Stride &&
+ "Expected dimensions to be sorted");
+ }
+ }
+
+ void dump(llvm::raw_ostream &OS) {
+ OS << "ArrayInfo:\n";
+ for (unsigned I = 0; I < Dims.size(); I++) {
+ // clang-format off
+ OS << " Dim " << I
+ << "\t" << STR_AND_VAL(Dims[I].Offset)
+ << "\t" << STR_AND_VAL(Dims[I].Size)
+ << "\t" << STR_AND_VAL(Dims[I].Stride)
+ << "\t" << STR_AND_VAL(Dims[I].Extent)
+ << "\n";
+ // clang-format on
+ }
+ // clang-format off
+ OS << " "
+ << " " << STR_AND_VAL(RawMemoryAddr)
+ << " " << OPT_STR_AND_VAL(RawMemorySize)
+ << " " << STR_AND_VAL(ElementSize)
+ << "\n";
+ // clang-format on
+ };
+
+ /// See the llvm-project/offload/test/offloading/non_contiguous_update.cpp
+ /// test for examples.
+ std::optional<NonContigDescTy> generateNonContigCopyDesc(ident_t *Loc) {
+ NonContigDescTy CopyDesc;
+ CopyDesc.Dims.reserve(Dims.size() + 1);
+
+ for (int I = Dims.size() - 1; I >= 0; I--) {
+ auto const &Dim = Dims[I];
+ if (Dim.Size < 0) {
+ ODBG(ADT_Descriptor)
+ << "Dim size missing, cannot build copy descriptor";
+ return std::nullopt;
+ }
+ CopyDesc.Dims.push_back({});
+ auto &LastDim = CopyDesc.Dims.back();
+ LastDim.Count = Dim.Size;
+ LastDim.Stride = Dim.Stride * ElementSize;
+ LastDim.Offset = Dim.Offset * LastDim.Stride;
+ }
+
+ CopyDesc.Dims.push_back({});
+ auto &LastDim = CopyDesc.Dims.back();
+ LastDim.Count = ElementSize;
+ LastDim.Offset = 0;
+ LastDim.Stride = 1;
+
+ return CopyDesc;
+ }
+};
+
+struct MaterializedMemRefDesc {
+ void *allocatedPtr;
+ void *alignedPtr;
+ uint64_t offset;
+ int64_t elementSize;
+ unsigned char rank;
+ const uint64_t *sizes;
+ const uint64_t *strides;
+};
+
+void dump(const MaterializedMemRefDesc &Desc, llvm::raw_ostream &OS) {
+ // clang-format off
+ OS
+ << " " << STR_AND_VAL(Desc.allocatedPtr)
+ << " " << STR_AND_VAL(Desc.alignedPtr)
+ << " " << STR_AND_VAL(Desc.offset)
+ << " " << STR_AND_VAL(Desc.elementSize)
+ << " " << STR_AND_VALI(Desc.rank)
+ << "\n";
+ for (unsigned I = 0; I < Desc.rank; I++) {
+ OS << "Dim " << I
+ << " " << STR_AND_VAL(Desc.sizes[I])
+ << " " << STR_AND_VAL(Desc.strides[I])
+ << "\n";
+ }
+ // clang-format on
+}
+
+void dump(const AccDataDescOpenACC &Desc, llvm::raw_ostream &OS) {
+ // clang-format off
+ OS
+ << " " << STR_AND_VAL(Desc.Base.Version)
+ << " " << STR_AND_VALI(Desc.Rank)
+ << " " << STR_AND_VAL(Desc.ElementSize)
+ << "\n";
+ for (unsigned I = 0; I < Desc.Rank; I++) {
+ OS << "Dim " << I
+ << " " << STR_AND_VAL(Desc.LowerBounds[I])
+ << " " << STR_AND_VAL(Desc.UpperBounds[I])
+ << " " << STR_AND_VAL(Desc.Extents[I])
+ << " " << STR_AND_VAL(Desc.StridesInBytes[I])
+ << " " << STR_AND_VAL(Desc.StartIndices[I])
+ << "\n";
+ }
+ // clang-format on
+}
+
+template <class... Ts> struct overloads : Ts... {
+ using Ts::operator()...;
+};
+template <class... Ts> overloads(Ts...) -> overloads<Ts...>;
+
+std::string asyncToString(int64_t Async) {
+ if (Async >= 0) {
+ return "STREAM(" + std::to_string(Async) + ")";
+ } else if (Async == AccAsyncSync) {
+ return "SYNC";
+ } else if (Async == AccAsyncDefault) {
+ return "DEFAULT";
+ } else if (Async == AccAsyncNoval) {
+ return "NOVAL";
+ } else {
+ return "UNKNOWN";
+ }
+}
+
+std::string mapTypeToString(int64_t Type) {
+ std::stringstream SS;
+
+ if (Type & TGT_ACC_MAPTYPE_TO)
+ SS << "TO ";
+ if (Type & TGT_ACC_MAPTYPE_FROM)
+ SS << "FROM ";
+ if (Type & TGT_ACC_MAPTYPE_FINALIZE)
+ SS << "DELETE ";
+ if (Type & TGT_ACC_MAPTYPE_PTR_AND_OBJ)
+ SS << "PTR_AND_OBJ ";
+ if (Type & TGT_ACC_MAPTYPE_RETURN_PARAM)
+ SS << "RETURN_PARAM ";
+ if (Type & TGT_ACC_MAPTYPE_PRIVATE)
+ SS << "PRIVATE ";
+ if (Type & TGT_ACC_MAPTYPE_LITERAL)
+ SS << "LITERAL ";
+ if (Type & TGT_ACC_MAPTYPE_IMPLICIT)
+ SS << "IMPLICIT ";
+ if (Type & TGT_ACC_MAPTYPE_DEVPTR)
+ SS << "DEVPTR ";
+ if (Type & TGT_ACC_MAPTYPE_MANAGED_DEVPTR)
+ SS << "MANAGED_DEVPTR ";
+ if (Type & TGT_ACC_MAPTYPE_NO_CREATE)
+ SS << "NO_CREATE ";
+ if (Type & TGT_ACC_MAPTYPE_GANG_PRIVATE)
+ SS << "GANG_PRIVATE ";
+ if (Type & TGT_ACC_MAPTYPE_WORKER_PRIVATE)
+ SS << "WORKER_PRIVATE ";
+ if (Type & TGT_ACC_MAPTYPE_VECTOR_PRIVATE)
+ SS << "VECTOR_PRIVATE ";
+ if (Type & TGT_ACC_MAPTYPE_INIT_ZERO)
+ SS << "INIT_ZERO ";
+ if (Type & TGT_ACC_MAPTYPE_DEVICE_RESIDENT)
+ SS << "DEVICE_RESIDENT ";
+ if (Type & TGT_ACC_MAPTYPE_IF_PRESENT)
+ SS << "IF_PRESENT ";
+ if (Type & TGT_ACC_MAPTYPE_PRESENT)
+ SS << "PRESENT ";
+ if (Type & TGT_ACC_MAPTYPE_DESCRIPTOR)
+ SS << "DESCRIPTOR ";
+ if (Type & TGT_ACC_MAPTYPE_REDUCTION)
+ SS << "REDUCTION ";
+
+ std::string Str = SS.str();
+ if (Str.empty())
+ return "(none)";
+ else
+ // Remove trailing space.
+ Str.resize(Str.size() - 1);
+
+ return Str;
+}
+
+enum class AccCopyOutType { Always, OnDelete, Never };
+enum class AccRefCountingType { Dynamic, Structured };
+
+struct PostProcessingInfo {
+ /// The target pointer information.
+ TargetPointerResultTy TPR;
+ int64_t DataSize;
+ bool ShouldRestoreShadow;
+ bool ShouldDelete;
+};
+
+struct KernelArgsMappingInfoTy {
+ AccKernelArgsTy &KernelArgs;
+
+ // Memory needed for launch
+ void addLaunchAlloc(void *Alloc) { LaunchAllocs.push_back(Alloc); }
+ SmallVector<void *> LaunchAllocs;
+
+ // Arguments
+ void addArg(void *Arg) { Args.push_back(Arg); }
+ SmallVector<void *> Args;
+ SmallVector<void *> Ptrs;
+
+ KernelLaunchParamsTy getLaunchArgs() {
+ assert(Ptrs.size() == 0);
+
+ if (Args.size() == 0)
+ return KernelLaunchParamsTy{};
+
+ unsigned NumArgs = Args.size();
+ Ptrs.resize(NumArgs);
+ for (uint32_t I = 0; I < NumArgs; ++I)
+ Ptrs[I] = &Args[I];
+ return KernelLaunchParamsTy{NumArgs, &Ptrs[0]};
+ }
+};
+
+[[nodiscard]] int accPostProcessingTargetDataEnd(DeviceTy *Device,
+ PostProcessingInfo *Info) {
+ // This will make sure we delete it when we exit the function.
+ std::unique_ptr<PostProcessingInfo> InfoDeleter(Info);
+
+ int Ret = OFFLOAD_SUCCESS;
+
+ assert(!Info->TPR.isHostPointer());
+
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device->getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
+
+ // We cannot use a lock guard because we may end up delete the mutex.
+ // We also explicitly unlocked the entry after it was put in the EntriesInfo
+ // so it can be reused.
+ Info->TPR.getEntry()->lock();
+ auto *Entry = Info->TPR.getEntry();
+
+ // Any data-end post-processor may delete, but only the last
+ // DataEndThreadCount holder with a zero refcount actually does. A
+ // non-deleting data-end (e.g. present around an async kernel) and a
+ // deleting one (outer data region) can be queued on different AsyncInfo
+ // objects and complete in either order.
+ bool DelEntry = true;
+ const bool IsNotLastUser = Entry->decDataEndThreadCount() != 0;
+ if (Entry->getTotalRefCount() != 0 || IsNotLastUser) {
+ ODBG(ADT_Mapping) << "Deferring delete: IsNotLastUser=" << IsNotLastUser
+ << " TotalRefCount=" << Entry->getTotalRefCount();
+ // The thread is not in charge of deletion anymore. Give up access
+ // to the HDTT map and unset the deletion flag.
+ HDTTMap.destroy();
+ DelEntry = false;
+ }
+
+ if (Info->ShouldRestoreShadow) {
+ Entry->foreachShadowPointerInfo([&](const ShadowPtrInfoTy &ShadowPtr) {
+ ODBG(ADT_Mapping) << "Restoring host shadow "
+ << (void *)ShadowPtr.HstPtrAddr
+ << " to its original content (" << ShadowPtr.PtrSize
+ << " bytes)";
+ std::memcpy(ShadowPtr.HstPtrAddr, ShadowPtr.HstPtrContent.data(),
+ ShadowPtr.PtrSize);
+ return OFFLOAD_SUCCESS;
+ });
+ }
+
+ // Give up the lock as we either don't need it anymore (e.g., done with
+ // TPR), or erase TPR.
+ Info->TPR.setEntry(nullptr);
+
+ if (!DelEntry)
+ return Ret;
+
+ Ret = Device->getMappingInfo().eraseMapEntry(HDTTMap, Entry, Info->DataSize);
+ // Entry is already remove from the map, we can unlock it now.
+ HDTTMap.destroy();
+ Ret |= Device->getMappingInfo().deallocTgtPtrAndEntry(Entry, Info->DataSize);
+ if (Ret != OFFLOAD_SUCCESS)
+ REPORT_FATAL() << "Deallocating data from device failed.";
+
+ return OFFLOAD_SUCCESS;
+}
+
+template <typename SizeTy>
+void handleSingleDataEnd(ident_t *Loc, void *ArgBasePtr, void *ArgPtr,
+ SizeTy ArgSize, bool ForceDelete, bool IsNoCreate,
+ AccCopyOutType CopyType, AccRefCountingType MapType,
+ AsyncInfoTy &AsyncInfo, DeviceTy &Device) {
+ int64_t DataSize;
+ if constexpr (std::is_same<SizeTy, int64_t>::value) {
+ DataSize = ArgSize;
+ } else if constexpr (std::is_same<SizeTy, NonContigDescTy &>::value) {
+ DataSize = ArgSize.getAllocSize();
+ } else {
+ static_assert(false);
+ }
+
+ FUNC_LOGGER();
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ ArgPtr, DataSize, /*UpdateRefCount=*/true,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured, IsNoCreate,
+ ForceDelete,
+ /*FromDataEnd=*/true);
+ if (!TPR.isPresent()) {
+ ODBG(ADT_Mapping) << "Mapping does not exist: "
+ << (IsNoCreate ? "is no_create" : "error");
+ if (Pedantic && !IsNoCreate)
+ REPORT_FATAL() << "Device mapping does not exist at " << Loc;
+ return;
+ }
+
+ void *HstPtrBegin = ArgPtr;
+ void *TgtPtrBegin = TPR.TargetPointer;
+ ODBG(ADT_Mapping) << "There are " << DataSize
+ << " bytes allocated at target address " << TgtPtrBegin
+ << " - is" << (TPR.Flags.IsLast ? "" : " not") << " last";
+
+ bool ShouldDelete = ForceDelete || TPR.Flags.IsLast;
+ bool ShouldCopyOut = CopyType == AccCopyOutType::Always ||
+ (CopyType == AccCopyOutType::OnDelete && ShouldDelete);
+ if (ShouldCopyOut) {
+ ODBG(ADT_Mapping) << "Moving " << DataSize << " bytes (tgt:" << TgtPtrBegin
+ << ") -> (hst:" << HstPtrBegin << ")";
+ int Ret;
+ if constexpr (std::is_same<SizeTy, int64_t>::value) {
+ Ret = Device.retrieveData(HstPtrBegin, TgtPtrBegin, DataSize, AsyncInfo,
+ TPR.getEntry());
+ } else if constexpr (std::is_same<SizeTy, NonContigDescTy &>::value) {
+ Ret = Device.retrieveNonContigData(HstPtrBegin, TgtPtrBegin, ArgSize,
+ AsyncInfo, TPR.getEntry());
+ }
+ if (Ret != OFFLOAD_SUCCESS)
+ REPORT_FATAL() << "Failed to transfer data from device at " << Loc;
+ }
+
+ ODBG(ADT_Mapping) << "Queueing up post processing";
+
+ // TODO We may want to have a more intricate system for queueing up post
+ // processing. In OpenACC, we could potentially queue up a lot of stream
+ // operations before syncing, and we only execute these post processing
+ // functions after we sync. This would leave a lot of deallocation and
+ // unmapping queued for post processing but never happening because the
+ // operations in the stream are continuing to execute while we add more post
+ // processing funcitons which we never execute.
+ //
+ // One option here is to use Device.enqueueHostCall, however, for example, for
+ // CUDA, having cuFree's etc in a function executed in the stream is not
+ // supported (the context in that thread is invalid for calling cuda
+ // functions). Instead, we can have the host call "notify" that we can execute
+ // specific post processing functions, and we execute them at some point
+ // during execution on the normal threads.
+ auto *PostProcessingPtr = new PostProcessingInfo{std::move(TPR), DataSize,
+ ShouldCopyOut, ShouldDelete};
+ PostProcessingPtr->TPR.getEntry()->unlock();
+ AsyncInfo.addPostProcessingFunction([=, Device = &Device]() -> int {
+ return accPostProcessingTargetDataEnd(Device, PostProcessingPtr);
+ });
+}
+
+struct DescAndMemMappingInfoTy {
+ DescMappingInfoTy Desc;
+ std::optional<MemMappingInfoTy> Memory;
+};
+
+/// Whether a descriptor's storage is part of a larger mapped object, as a
+/// derived-type member descriptor is part of its parent.
+///
+/// Such a descriptor is placed on the device by the parent's own mapping, which
+/// also owns its lifetime, so it must be neither reference counted nor copied
+/// on its own account: a reference would put a clause naming only the member in
+/// charge of part of the parent's lifetime, and a copy would overwrite the
+/// device descriptor with a host pointer.
+///
+/// A descriptor that owns its mapping, as a local allocatable or pointer
+/// usually does, is reference counted as any other mapping. The two are
+/// indistinguishable when the parent holds nothing but the one descriptor, in
+/// which case the descriptor is treated as owning the mapping.
+/// A lookup can also report a neighbouring mapping, which two descriptors
+/// sitting next to each other on the stack routinely produce, so the descriptor
+/// must be found wholly inside the entry and not merely overlapping it.
+bool isDescriptorInLargerMapping(const HostDataToTargetTy *Entry,
+ void *DescriptorAddr, int64_t DescriptorSize) {
+ if (!Entry)
+ return false;
+ auto Begin = reinterpret_cast<uintptr_t>(DescriptorAddr);
+ auto End = Begin + static_cast<uintptr_t>(DescriptorSize);
+ if (Entry->HstPtrBegin > Begin || Entry->HstPtrEnd < End)
+ return false;
+ return Entry->HstPtrBegin < Begin || Entry->HstPtrEnd > End;
+}
+
+struct ArgDescriptorsTy {
+
+ const Fortran::runtime::Descriptor *Flang = nullptr;
+ std::optional<MaterializedMemRefDesc> MemRef = std::nullopt;
+ const AccDataDescOpenACC *Acc = nullptr;
+
+ bool isNone() { return !Flang && !MemRef && !Acc; }
+
+ void verify() {
+ assert(!(Flang && MemRef));
+ assert(!isNone());
+ }
+
+ void dump(llvm::raw_ostream &OS) {
+ OS << "ArgDescriptorsTy:\n";
+ OS << "Flang:\n";
+ if (Flang) {
+ // TODO can we use OS somehow?
+ Flang->Dump(stderr);
+ } else {
+ OS << "(nil)\n";
+ }
+ OS << "MemRef:\n";
+ if (MemRef) {
+ ::dump(*MemRef, OS);
+ } else {
+ OS << "(nil)\n";
+ }
+ OS << "Acc:\n";
+ if (Acc) {
+ ::dump(*Acc, OS);
+ } else {
+ OS << "(nil)\n";
+ }
+ }
+
+ void collectAccBounds(ident_t *Loc, ArrayInfo &AI) {
+ assert(Acc);
+ if (AI.ElementSize <= 0) {
+ REPORT_FATAL() << Loc << "Invalid element size";
+ }
+
+ AI.Dims.reserve(Acc->Rank);
+ for (std::size_t I = 0; I < Acc->Rank; I++) {
+ AI.Dims.push_back({});
+ auto &ThisDim = AI.Dims.back();
+ long SizeFactor = 1;
+ if (Acc->StridesInBytes[I] % AI.ElementSize != 0) {
+ if (I == 0) {
+ // `stride` in AccArrayDim is meant to be multiplied by elementsize.
+ // But the stride of a sliced descriptor array might not be divisible
+ // by the current element size. So, reduce elementsize.
+ SizeFactor =
+ AI.ElementSize / std::gcd(AI.ElementSize, Acc->StridesInBytes[I]);
+ AI.ElementSize /= SizeFactor;
+ } else {
+ REPORT_FATAL() << Loc << "Invalid array stride";
+ }
+ }
+ ThisDim.Offset = Acc->LowerBounds[I];
+ ThisDim.Stride = Acc->StridesInBytes[I] / AI.ElementSize;
+ ThisDim.Size =
+ SizeFactor * (Acc->UpperBounds[I] - Acc->LowerBounds[I] + 1);
+
+ // TODO do we need this for offload? Can we always take Acc's extents?
+#if 0
+ if (Flang && Flang->GetDimension(i).Extent() != -1) {
+ // The extent of the OpenACC descriptor is the user-specified extent.
+ // But we need to pass the actual array extents.
+ ThisDim.extent = Flang->GetDimension(i).Extent();
+ } else {
+ ThisDim.extent = Acc->Extents[i];
+ }
+#else
+ ThisDim.Extent = Acc->Extents[I];
+#endif
+ }
+ }
+
+ void collectFlangBounds(ident_t *Loc, ArrayInfo &AI) {
+ assert(Flang);
+ if (AI.ElementSize <= 0) {
+ REPORT_FATAL() << Loc << "Invalid element size";
+ }
+
+ AI.Dims.reserve(Flang->rank());
+ AI.ElementSize = Flang->ElementBytes();
+ for (int I = 0; I < Flang->rank(); I++) {
+ AI.Dims.push_back({});
+ auto &ThisDim = AI.Dims.back();
+ auto &FlangDim = Flang->GetDimension(I);
+ long SizeFactor = 1;
+ if (FlangDim.ByteStride() % AI.ElementSize != 0) {
+ if (I == 0) {
+ SizeFactor =
+ AI.ElementSize / std::gcd(AI.ElementSize, FlangDim.ByteStride());
+ AI.ElementSize /= SizeFactor;
+ } else {
+ REPORT_FATAL() << Loc << "Invalid array stride";
+ }
+ }
+ ThisDim.Offset = 0;
+ ThisDim.Stride = FlangDim.ByteStride() / AI.ElementSize;
+ ThisDim.Size = SizeFactor * FlangDim.Extent();
+ }
+ }
+
+ using LiteralArg = void *;
+ using ArgMappingInfoTy =
+ std::variant<DescAndMemMappingInfoTy, MemMappingInfoTy, LiteralArg>;
+ using ArgMappingInfosTy = std::vector<ArgMappingInfoTy>;
+
+ ArgMappingInfosTy getMappingInfos(ident_t *Loc, void *Ptr) {
+ if (Flang) {
+ auto DMI = DescMappingInfoTy{Flang->SizeInBytes(),
+ offsetof(CFI_cdesc_t, base_addr)};
+ if (!Flang->IsAllocated()) {
+ ODBG() << "Is not allocated - nothing to map.";
+ ArgMappingInfosTy MIs;
+ MIs.emplace_back(DescAndMemMappingInfoTy{DMI, std::nullopt});
+ return MIs;
+ }
+
+ ArrayInfo AI;
+ AI.ElementSize = Flang->ElementBytes();
+ if (AI.ElementSize == 0 && Acc) {
+ AI.ElementSize = Acc->ElementSize;
+ }
+ AI.setPtr(Flang->OffsetElement(0));
+
+ if (Acc) {
+ collectAccBounds(Loc, AI);
+ } else {
+ collectFlangBounds(Loc, AI);
+ }
+ AI.normalize();
+ AI.computeSizeFromDims(Loc);
+
+ auto MMI = MemMappingInfoTy{};
+ MMI.RawMemoryPtr = AI.RawMemoryAddr;
+ MMI.RawMemoryBasePtr = Flang->OffsetElement(0);
+ MMI.RawMemorySize = AI.RawMemorySize;
+ MMI.CopyDesc = AI.generateNonContigCopyDesc(Loc);
+
+ ArgMappingInfosTy MIs;
+ MIs.emplace_back(DescAndMemMappingInfoTy{DMI, std::move(MMI)});
+ return MIs;
+ } else if (MemRef) {
+ if (Acc) {
+ REPORT_FATAL() << Loc << "Unsupported: MemRef with OpenACC bounds";
+ }
+
+ ArgMappingInfosTy MIs;
+
+ uint64_t Extent = 1LL;
+ for (ssize_t I = (ssize_t)MemRef->rank - 1; I >= 0; I--) {
+ Extent *= MemRef->sizes[I];
+ if (Extent != MemRef->strides[I] * MemRef->sizes[I]) {
+ REPORT_FATAL() << Loc << "Invalid memref descriptor";
+ }
+ }
+ Extent *= MemRef->elementSize;
+
+ {
+ auto MMI = MemMappingInfoTy{};
+ MMI.RawMemoryPtr = MemRef->allocatedPtr;
+ MMI.RawMemoryBasePtr = MemRef->allocatedPtr;
+ MMI.RawMemorySize = Extent;
+ MIs.push_back(std::move(MMI));
+ }
+ {
+ auto MMI = MemMappingInfoTy{};
+ MMI.RawMemoryPtr = MemRef->alignedPtr;
+ MMI.RawMemoryBasePtr = MemRef->allocatedPtr;
+ MMI.RawMemorySize = Extent;
+ MIs.push_back(std::move(MMI));
+ }
+
+ MIs.push_back(reinterpret_cast<void *>(MemRef->offset));
+
+ for (size_t I = 0; I < MemRef->rank; I++) {
+ MIs.push_back(reinterpret_cast<void *>(MemRef->sizes[I]));
+ MIs.push_back(reinterpret_cast<void *>(MemRef->strides[I]));
+ }
+
+ return MIs;
+ } else if (Acc) {
+ ArrayInfo AI;
+ AI.ElementSize = Acc->ElementSize;
+ AI.setPtr(Ptr);
+ collectAccBounds(Loc, AI);
+ AI.normalize();
+ AI.computeSizeFromDims(Loc);
+
+ auto MMI = MemMappingInfoTy{};
+ MMI.RawMemoryPtr = AI.RawMemoryAddr;
+ MMI.RawMemoryBasePtr = Ptr;
+ MMI.RawMemorySize = AI.RawMemorySize;
+ MMI.CopyDesc = AI.generateNonContigCopyDesc(Loc);
+
+ ArgMappingInfosTy MIs;
+ MIs.push_back(std::move(MMI));
+ return MIs;
+ } else {
+ REPORT_FATAL() << Loc << "Unknown case.";
+ abort();
+ }
+ }
+
+ void dataBeginPrivate(ident_t *Loc, void *ArgPtr, int64_t ArgSize,
+ bool HasFlagTo, DeviceTy &Device,
+ AsyncInfoTy &AsyncInfo,
+ MappingInfoTy::HDTTMapAccessorTy &HDTTMap,
+ KernelArgsMappingInfoTy &KI) {
+ assert(!MemRef);
+
+ if (Flang) {
+ assert(ArgPtr);
+ // The kernel receives a descriptor, so the private copy is a private
+ // descriptor whose base address points at a private copy of the payload.
+ ArrayInfo AI;
+ AI.ElementSize = Flang->ElementBytes();
+ AI.setPtr(Flang->raw().base_addr);
+ collectFlangBounds(Loc, AI);
+ AI.normalize();
+ AI.computeSizeFromDims(Loc);
+
+ void *PrivateMemory = nullptr;
+ if (AI.RawMemorySize && *AI.RawMemorySize) {
+ PrivateMemory =
+ Device.allocData(*AI.RawMemorySize, nullptr, TARGET_ALLOC_DEVICE);
+ ODBG(ADT_Interface)
+ << "Flang private: dataSize=" << *AI.RawMemorySize
+ << " hostData=" << AI.RawMemoryAddr << " at " << PrivateMemory;
+ if (HasFlagTo)
+ Device.submitData(PrivateMemory, AI.RawMemoryAddr, *AI.RawMemorySize,
+ AsyncInfo, /*Entry=*/nullptr, &HDTTMap);
+ }
+
+ size_t DescSize = Flang->SizeInBytes();
+ void *DevDesc = Device.allocData(DescSize, nullptr, TARGET_ALLOC_DEVICE);
+ Device.submitData(DevDesc,
+ const_cast<void *>(static_cast<const void *>(Flang)),
+ DescSize, AsyncInfo, /*Entry=*/nullptr, &HDTTMap);
+
+ if (PrivateMemory) {
+ // Normalization can move the start of the payload away from the
+ // descriptor's own base address, so shift the device address by the
+ // same amount rather than pointing at the payload's first byte.
+ ptrdiff_t Offset = reinterpret_cast<intptr_t>(Flang->raw().base_addr) -
+ reinterpret_cast<intptr_t>(AI.RawMemoryAddr);
+ void *&Buf = AsyncInfo.getVoidPtrLocation();
+ Buf = static_cast<char *>(PrivateMemory) + Offset;
+ Device.submitData(
+ static_cast<char *>(DevDesc) + offsetof(CFI_cdesc_t, base_addr),
+ &Buf, sizeof(void *), AsyncInfo, /*Entry=*/nullptr, &HDTTMap);
+ KI.addLaunchAlloc(PrivateMemory);
+ }
+
+ KI.addArg(DevDesc);
+ KI.addLaunchAlloc(DevDesc);
+ } else if (Acc) {
+ assert(ArgPtr);
+ ArrayInfo AI;
+ AI.ElementSize = Acc->ElementSize;
+ AI.setPtr(ArgPtr);
+ collectAccBounds(Loc, AI);
+ AI.normalize();
+ AI.computeSizeFromDims(Loc);
+
+ assert(AI.RawMemorySize);
+ size_t DataSize = *AI.RawMemorySize;
+ void *HostData = AI.RawMemoryAddr;
+ ptrdiff_t Offset = reinterpret_cast<intptr_t>(HostData) -
+ reinterpret_cast<intptr_t>(ArgPtr);
+
+ ODBG(ADT_Interface) << "ACC firstprivate (partial): dataSize=" << DataSize
+ << " hostData=" << HostData << " (base=" << ArgPtr
+ << ")";
+
+ void *PrivateMemory =
+ Device.allocData(DataSize, nullptr, TARGET_ALLOC_DEVICE);
+ Device.submitData(PrivateMemory, HostData, DataSize, AsyncInfo,
+ /*Entry=*/nullptr, &HDTTMap);
+ KI.addArg(static_cast<char *>(PrivateMemory) - Offset);
+ KI.addLaunchAlloc(PrivateMemory);
+ } else {
+ REPORT_FATAL() << Loc << "Unknown descriptor type for private variable";
+ }
+ }
+
+ void dataBeginDevPtr(ident_t *Loc, DeviceTy &Device, AsyncInfoTy &AsyncInfo,
+ MappingInfoTy::HDTTMapAccessorTy &HDTTMap,
+ KernelArgsMappingInfoTy &KI) {
+ assert(Flang && !MemRef);
+ size_t DescSize = Flang->SizeInBytes();
+ void *DevDesc = Device.allocData(DescSize, nullptr, TARGET_ALLOC_DEVICE);
+ Device.submitData(DevDesc,
+ const_cast<void *>(static_cast<const void *>(Flang)),
+ DescSize, AsyncInfo, /*Entry=*/nullptr, &HDTTMap);
+ KI.addArg(DevDesc);
+ KI.addLaunchAlloc(DevDesc);
+ }
+
+ void dataBegin(ident_t *Loc, void *ArgPtr, void *DescriptorAddr,
+ void *&ParentAllocation, bool IsPtrAndObj, bool MapDescriptor,
+ char *ArgName, bool HasFlagTo, bool IsNoCreate,
+ AccRefCountingType MapType, AsyncInfoTy &AsyncInfo,
+ DeviceTy &Device, MappingInfoTy::HDTTMapAccessorTy &HDTTMap,
+ KernelArgsMappingInfoTy *KI) {
+ auto AddArg = [&](TargetPointerResultTy &TPR, void *TgtArg, void *HstArg) {
+ if (KI) {
+ if (TPR.isPresent()) {
+ KI->addArg(TgtArg);
+ } else {
+ assert(IsNoCreate);
+ KI->addArg(HstArg);
+ }
+ }
+ };
+ auto MapWithDesc = [&](MemMappingInfoTy &MemInfo, void *BasePtr,
+ bool IsParam) -> void * {
+ assert(MemInfo.CopyDesc);
+ ODBG() << "Will use non-contig copy.";
+
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, MemInfo.RawMemoryPtr, MemInfo.RawMemoryBasePtr, 0,
+ &*MemInfo.CopyDesc, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ /*HasPresentModifier=*/false,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ if (IsParam)
+ AddArg(TPR,
+ reinterpret_cast<void *>(
+ (reinterpret_cast<intptr_t>(TPR.TargetPointer) -
+ MemInfo.getBaseDelta())),
+ MemInfo.RawMemoryPtr);
+ return TPR.TargetPointer;
+ };
+
+ auto MapInfos = getMappingInfos(Loc, ArgPtr);
+ auto MemCase = [&](MemMappingInfoTy &MemInfo) {
+ if (MemInfo.RawMemorySize) {
+ if (MemInfo.CopyDesc) {
+ MapWithDesc(MemInfo, MemInfo.RawMemoryBasePtr, true);
+ } else {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, MemInfo.RawMemoryPtr, MemInfo.RawMemoryBasePtr, 0,
+ (int64_t)*MemInfo.RawMemorySize, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ /*HasPresentModifier=*/false,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ AddArg(TPR, TPR.TargetPointer, MemInfo.RawMemoryPtr);
+ }
+ } else {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, MemInfo.RawMemoryPtr, MemInfo.RawMemoryBasePtr, 0,
+ (int64_t)*MemInfo.RawMemorySize, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ /*HasPresentModifier=*/true,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ AddArg(TPR, TPR.TargetPointer, MemInfo.RawMemoryPtr);
+ }
+ };
+ auto DescAndMemCase = [&](DescAndMemMappingInfoTy &MapInfo) {
+ if (!MapDescriptor) {
+ ODBG() << "Mapping mem only";
+ if (MapInfo.Memory)
+ MemCase(*MapInfo.Memory);
+ return;
+ }
+
+ ODBG() << "Mapping desc and mem";
+ auto &DescInfo = MapInfo.Desc;
+
+ void *DescTgtPtr = nullptr;
+ if (!ParentAllocation)
+ ParentAllocation = DescriptorAddr;
+
+ // Scoped so the probe releases the entry before it is mapped below.
+ bool DescInLargerMapping;
+ {
+ LookupResult LR = Device.getMappingInfo().lookupMapping(
+ HDTTMap, DescriptorAddr, DescInfo.DescriptorSize);
+ DescInLargerMapping = isDescriptorInLargerMapping(
+ LR.TPR.getEntry(), DescriptorAddr, DescInfo.DescriptorSize);
+ }
+
+ {
+ // Always copy the descriptor to device. It is needed regardless of the
+ // user-specified TO/FROM, and regardless of whether no_create is on or
+ // not as the no_create can refer to the raw memory in the descriptor.
+ // A descriptor that belongs to a larger mapping is only looked up here,
+ // to get an address to attach through.
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, DescriptorAddr, DescriptorAddr, 0,
+ (int64_t)DescInfo.DescriptorSize, ArgName,
+ /*HasFlagTo=*/!DescInLargerMapping,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/!DescInLargerMapping, /*HasCloseModifier=*/false,
+ /*HasPresentModifier=*/false,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ /*IsNoCreate=*/false, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ DescTgtPtr = TPR.TargetPointer;
+ AddArg(TPR, DescTgtPtr, DescriptorAddr);
+ }
+
+ void *MemTgtPtr = nullptr;
+ if (MapInfo.Memory) {
+ auto &MemInfo = *MapInfo.Memory;
+ void *BasePtr =
+ static_cast<char *>(DescriptorAddr) + DescInfo.RawMemoryPtrOffset;
+ if (MemInfo.RawMemorySize) {
+ if (MemInfo.CopyDesc) {
+ MemTgtPtr = MapWithDesc(MemInfo, BasePtr, false);
+ } else {
+ TargetPointerResultTy TPR =
+ Device.getMappingInfo().getTargetPointer(
+ HDTTMap, MemInfo.RawMemoryPtr, BasePtr, 0,
+ (int64_t)*MemInfo.RawMemorySize, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ /*HasPresentModifier=*/false,
+ /*HasHoldModifier=*/MapType ==
+ AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ MemTgtPtr = TPR.TargetPointer;
+ }
+ } else {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, MemInfo.RawMemoryPtr, BasePtr, 0,
+ (int64_t)*MemInfo.RawMemorySize, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ /*HasPresentModifier=*/true,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ MemTgtPtr = TPR.TargetPointer;
+ }
+
+ if (MemTgtPtr && DescTgtPtr) {
+ LookupResult DescLR = Device.getMappingInfo().lookupMapping(
+ HDTTMap, DescriptorAddr, DescInfo.DescriptorSize);
+ auto *DescEntry = DescLR.TPR.getEntry();
+ if (DescEntry) {
+ uintptr_t TgtDescBase =
+ DescEntry->TgtPtrBegin +
+ (reinterpret_cast<uintptr_t>(DescriptorAddr) -
+ DescEntry->HstPtrBegin);
+ void **TgtPtrAddr = reinterpret_cast<void **>(
+ TgtDescBase + DescInfo.RawMemoryPtrOffset);
+ void **HstPtrAddr = reinterpret_cast<void **>(
+ reinterpret_cast<uintptr_t>(DescriptorAddr) +
+ DescInfo.RawMemoryPtrOffset);
+
+ void *HstBaseAddr = *HstPtrAddr;
+ void *TgtPteeBase = reinterpret_cast<void *>(
+ reinterpret_cast<uintptr_t>(MemTgtPtr) +
+ (reinterpret_cast<uintptr_t>(HstBaseAddr) -
+ reinterpret_cast<uintptr_t>(MemInfo.RawMemoryPtr)));
+
+ if (DescEntry->addShadowPointer(
+ ShadowPtrInfoTy{HstPtrAddr, TgtPtrAddr, TgtPteeBase,
+ static_cast<int64_t>(sizeof(void *))})) {
+ ODBG() << "DescAndMemCase attach: device field " << TgtPtrAddr
+ << " -> " << TgtPteeBase;
+ void *&Buf = AsyncInfo.getVoidPtrLocation();
+ Buf = TgtPteeBase;
+ Device.submitData(TgtPtrAddr, &Buf, sizeof(void *), AsyncInfo,
+ DescEntry, &HDTTMap);
+ DescEntry->addEventIfNecessary(Device, AsyncInfo);
+ }
+ }
+ }
+ }
+ };
+ auto LiteralCase = [&](void *Literal) {
+ if (KI)
+ KI->addArg(Literal);
+ };
+ for (auto &MapInfo : MapInfos) {
+ std::visit(overloads{DescAndMemCase, MemCase, LiteralCase}, MapInfo);
+ }
+ }
+
+ void dataEnd(ident_t *Loc, void *ArgPtr, void *DescriptorAddr,
+ void *ParentAllocation, int64_t ArgType, bool MapDescriptor,
+ bool ForceDelete, bool IsNoCreate, AccCopyOutType CopyType,
+ AccRefCountingType MapType, AsyncInfoTy &AsyncInfo,
+ DeviceTy &Device) {
+ auto MapWithDesc = [&](MemMappingInfoTy &MemInfo, void *BasePtr) {
+ if (MemInfo.CopyDesc) {
+ ODBG(ADT_Mapping) << "Will use non-contig copy.";
+
+ handleSingleDataEnd<NonContigDescTy &>(
+ Loc, BasePtr, MemInfo.RawMemoryPtr, *MemInfo.CopyDesc, ForceDelete,
+ IsNoCreate, CopyType, MapType, AsyncInfo, Device);
+ }
+ };
+
+ auto MapInfos = getMappingInfos(Loc, ArgPtr);
+ auto MemCase = [&](MemMappingInfoTy &MemInfo) {
+ if (MemInfo.RawMemorySize) {
+ if (MemInfo.CopyDesc) {
+ MapWithDesc(MemInfo, MemInfo.RawMemoryBasePtr);
+ } else {
+ handleSingleDataEnd<int64_t>(
+ Loc, MemInfo.RawMemoryBasePtr, MemInfo.RawMemoryPtr,
+ *MemInfo.RawMemorySize, ForceDelete, IsNoCreate, CopyType,
+ MapType, AsyncInfo, Device);
+ }
+ } else {
+ handleSingleDataEnd<int64_t>(
+ Loc, MemInfo.RawMemoryBasePtr, MemInfo.RawMemoryPtr, 0, ForceDelete,
+ IsNoCreate, CopyType, MapType, AsyncInfo, Device);
+ }
+ };
+ auto DescAndMemCase = [&](DescAndMemMappingInfoTy &MapInfo) {
+ auto &DescInfo = MapInfo.Desc;
+
+ if (!MapDescriptor) {
+ if (MapInfo.Memory)
+ MemCase(*MapInfo.Memory);
+ return;
+ }
+
+ if (!ParentAllocation) {
+ ParentAllocation = DescriptorAddr;
+ }
+
+ // Mirrors dataBegin: a descriptor belonging to a larger mapping took no
+ // reference, so there is none to release. Releasing one would act on the
+ // enclosing object's entry, and a `finalize` would reset its count and
+ // end its data lifetime even though the clause named only the member.
+ // Scoped so the probe releases the entry before it is released below.
+ bool DescInLargerMapping;
+ {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ DescriptorAddr, DescInfo.DescriptorSize, /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
+ DescInLargerMapping = isDescriptorInLargerMapping(
+ TPR.getEntry(), DescriptorAddr, DescInfo.DescriptorSize);
+ }
+
+ if (!DescInLargerMapping)
+ handleSingleDataEnd<int64_t>(
+ Loc, DescriptorAddr, DescriptorAddr, DescInfo.DescriptorSize,
+ ForceDelete, IsNoCreate, CopyType, MapType, AsyncInfo, Device);
+
+ if (MapInfo.Memory) {
+ auto &MemInfo = *MapInfo.Memory;
+ void *BasePtr =
+ static_cast<char *>(DescriptorAddr) + DescInfo.RawMemoryPtrOffset;
+ if (MemInfo.RawMemorySize) {
+ if (MemInfo.CopyDesc) {
+ MapWithDesc(MemInfo, BasePtr);
+ } else {
+ handleSingleDataEnd<int64_t>(
+ Loc, BasePtr, MemInfo.RawMemoryPtr, *MemInfo.RawMemorySize,
+ ForceDelete, IsNoCreate, CopyType, MapType, AsyncInfo, Device);
+ }
+ } else {
+ handleSingleDataEnd<int64_t>(Loc, BasePtr, MemInfo.RawMemoryPtr, 0,
+ ForceDelete, IsNoCreate, CopyType,
+ MapType, AsyncInfo, Device);
+ }
+ }
+ };
+ auto LiteralCase = [&](void *Literal) {};
+ for (auto &MapInfo : MapInfos) {
+ std::visit(overloads{DescAndMemCase, MemCase, LiteralCase}, MapInfo);
+ }
+ }
+
+ void dataUpdate(ident_t *Loc, void *ArgPtr, int64_t ArgType,
+ AsyncInfoTy &AsyncInfo, DeviceTy &Device) {
+ const bool HasFlagTo = ArgType & TGT_ACC_MAPTYPE_TO;
+ const bool HasFlagFrom = ArgType & TGT_ACC_MAPTYPE_FROM;
+
+ auto LookupMapping = [&](void *HstPtr,
+ int64_t Size) -> TargetPointerResultTy {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ HstPtr, Size, /*UpdateRefCount=*/false, /*UseHoldRefCount=*/false,
+ /*MustContain=*/true);
+ if (!TPR.isPresent()) {
+ if (ArgType & TGT_ACC_MAPTYPE_IF_PRESENT) {
+ ODBG(ADT_Interface) << "Not present, if_present - skipping update.";
+ return TPR;
+ }
+ REPORT_FATAL() << "Device mapping does not exist for update at " << Loc;
+ }
+ return TPR;
+ };
+
+ auto DoContiguousUpdate = [&](void *HstPtr, int64_t Size) {
+ TargetPointerResultTy TPR = LookupMapping(HstPtr, Size);
+ if (!TPR.isPresent())
+ return;
+ void *TgtPtr = TPR.TargetPointer;
+ if (HasFlagTo) {
+ ODBG(ADT_Interface) << "Update TO: " << Size << " bytes hst:" << HstPtr
+ << " -> tgt:" << TgtPtr;
+ Device.submitData(TgtPtr, HstPtr, Size, AsyncInfo, TPR.getEntry());
+ }
+ if (HasFlagFrom) {
+ ODBG(ADT_Interface) << "Update FROM: " << Size
+ << " bytes tgt:" << TgtPtr << " -> hst:" << HstPtr;
+ Device.retrieveData(HstPtr, TgtPtr, Size, AsyncInfo, TPR.getEntry());
+ }
+ };
+
+ auto DoNonContigUpdate = [&](MemMappingInfoTy &MemInfo) {
+ ODBG(ADT_Interface) << "Will use non-contig update.";
+
+ int64_t AllocSize = MemInfo.CopyDesc->getAllocSize();
+
+ TargetPointerResultTy TPR =
+ LookupMapping(MemInfo.RawMemoryPtr, AllocSize);
+ if (!TPR.isPresent())
+ return;
+ void *TgtPtr = TPR.TargetPointer;
+ if (HasFlagTo) {
+ ODBG(ADT_Interface)
+ << "Non-contig update TO: hst:" << MemInfo.RawMemoryPtr
+ << " -> tgt:" << TgtPtr;
+ Device.submitNonContigData(TgtPtr, MemInfo.RawMemoryPtr,
+ *MemInfo.CopyDesc, AsyncInfo,
+ TPR.getEntry());
+ }
+ if (HasFlagFrom) {
+ ODBG(ADT_Interface) << "Non-contig update FROM: tgt:" << TgtPtr
+ << " -> hst:" << MemInfo.RawMemoryPtr;
+ Device.retrieveNonContigData(MemInfo.RawMemoryPtr, TgtPtr,
+ *MemInfo.CopyDesc, AsyncInfo,
+ TPR.getEntry());
+ }
+ };
+
+ auto MapInfos = getMappingInfos(Loc, ArgPtr);
+ for (auto &MapInfo : MapInfos) {
+ if (auto *DM = std::get_if<DescAndMemMappingInfoTy>(&MapInfo)) {
+ if (DM->Memory && DM->Memory->RawMemorySize) {
+ if (DM->Memory->CopyDesc) {
+ DoNonContigUpdate(*DM->Memory);
+ } else {
+ DoContiguousUpdate(DM->Memory->RawMemoryPtr,
+ *DM->Memory->RawMemorySize);
+ }
+ }
+ } else if (auto *MM = std::get_if<MemMappingInfoTy>(&MapInfo)) {
+ if (MM->RawMemorySize) {
+ if (MM->CopyDesc) {
+ DoNonContigUpdate(*MM);
+ } else {
+ DoContiguousUpdate(MM->RawMemoryPtr, *MM->RawMemorySize);
+ }
+ }
+ }
+ }
+ }
+};
+
+const uint64_t *getMemRefSizes(const MemRefDesc *Desc) {
+ return &Desc->sizes[0];
+}
+const uint64_t *getMemRefStrides(const MemRefDesc *Desc, unsigned Rank) {
+ return &Desc->sizes[0] + Rank;
+}
+
+ArgDescriptorsTy parseArgDescs(ident_t *Loc, const AccDataDesc *ArgDesc) {
+ ArgDescriptorsTy Descs;
+ if (!ArgDesc)
+ return Descs;
+
+ if (ArgDesc->Version & TGT_ACC_DESC_F18) {
+ Descs.Flang = reinterpret_cast<decltype(Descs.Flang)>(
+ ((const AccDataDescF18 *)ArgDesc)->FortranDescriptor);
+ }
+ if (ArgDesc->Version & TGT_ACC_DESC_MEMREF) {
+ const MemRefDesc *DescMemRef =
+ ((const AccDataDescMemRef *)ArgDesc)->MemRefDescriptor;
+ Descs.MemRef = MaterializedMemRefDesc{};
+ Descs.MemRef->allocatedPtr = DescMemRef->allocatedPtr;
+ Descs.MemRef->alignedPtr = DescMemRef->alignedPtr;
+ Descs.MemRef->offset = DescMemRef->offset;
+ Descs.MemRef->rank = ((const AccDataDescMemRef *)ArgDesc)->Rank;
+ Descs.MemRef->sizes = getMemRefSizes(DescMemRef);
+ Descs.MemRef->strides = getMemRefStrides(DescMemRef, Descs.MemRef->rank);
+ Descs.MemRef->elementSize =
+ ((const AccDataDescMemRef *)ArgDesc)->ElementSize;
+ }
+ if (ArgDesc->Version & TGT_ACC_DESC_OPENACC) {
+ int64_t DescPadding = 0;
+ if (Descs.Flang) {
+ DescPadding = sizeof(CFI_cdesc_t *);
+ } else if (Descs.MemRef) {
+ REPORT_FATAL() << Loc << "Unsupported: MemRef with OpenACC bounds";
+ } else {
+ DescPadding = 0;
+ }
+ Descs.Acc = reinterpret_cast<const AccDataDescOpenACC *>(
+ reinterpret_cast<const char *>(ArgDesc) + DescPadding);
+ }
+ return Descs;
+}
+
+ArgDescriptorsTy parseAndVerifyArgDescs(ident_t *Loc,
+ const AccDataDesc *ArgDesc) {
+ ArgDescriptorsTy Descs = parseArgDescs(Loc, ArgDesc);
+ ODBG_IF([&]() { Descs.dump(llvm::dbgs()); });
+ Descs.verify();
+ return Descs;
+}
+
+/// Whether an array descriptor takes part in the mapping of its data, rather
+/// than only describing which data to map.
+///
+/// A mapped descriptor is what lets device code reach the data through it, so
+/// it is needed for kernel arguments and for descriptors that sit inside a
+/// mapped parent object. Both of those, and structured regions, name the same
+/// descriptor again when the mapping ends, so the entry is released there.
+///
+/// An unstructured mapping has no such counterpart: its data is released by
+/// whichever construct or runtime API call later names that data, and that call
+/// carries its own descriptor - for a Fortran array section actual argument,
+/// one that lives only for the duration of the call. A descriptor mapped here
+/// would keep a reference count nothing ever drops, and once its storage is
+/// reused the stale entry breaks lookups of whatever now overlaps it.
+bool mapsDescriptor(AccRefCountingType MapType, bool IsPtrAndObj,
+ const KernelArgsMappingInfoTy *KI) {
+ return KI || IsPtrAndObj || MapType == AccRefCountingType::Structured;
+}
+
+/// Extent to map for an argument, or zero when only a descriptor can supply it.
+/// A `present` argument refers to data that is already mapped, so one byte is
+/// enough to find the entry containing it even when the compiler gave neither a
+/// size nor a descriptor.
+int64_t getMappingSize(int64_t ArgSize, int64_t ArgType,
+ const AccDataDesc *ArgDesc) {
+ if (ArgSize == 0 && !ArgDesc && (ArgType & TGT_ACC_MAPTYPE_PRESENT))
+ return 1;
+ return ArgSize;
+}
+
+void accTargetDataBegin(ident_t *Loc, void *ArgBasePtr, void *ArgPtr,
+ int64_t ArgSize, int64_t ArgType, char *ArgName,
+ AccDataDesc *ArgDesc, AccRefCountingType MapType,
+ AsyncInfoTy &AsyncInfo, DeviceTy &Device,
+ MappingInfoTy::HDTTMapAccessorTy &HDTTMap,
+ KernelArgsMappingInfoTy *KI = nullptr) {
+ // clang-format off
+ ODBG(ADT_Interface)
+ << "targetDataBegin "
+ << "ArgName=" << getNameFromMapping(ArgName) << ", "
+ << "ArgBasePtr=" << ArgBasePtr << ", "
+ << "ArgPtr=" << ArgPtr << ", "
+ << "ArgSize=" << ArgSize << ", "
+ << "ArgType=" << mapTypeToString(ArgType)
+ << " (" << llvm::format_hex(ArgType, 0) << "), "
+ << "ArgDesc=" << ArgDesc;
+ // clang-format on
+
+ // OpenACC 3.4: `if_present` is only valid on `host_data` and `update`
+ // directives.
+ assert(!(ArgType & TGT_ACC_MAPTYPE_IF_PRESENT));
+
+ // An absent optional argument or an unallocated allocatable has no storage,
+ // so there is nothing to map and a `present` clause naming it is ignored
+ // rather than diagnosed. The kernel still needs its argument slot, and the
+ // null there is what makes `present()` false on the device. Private and
+ // literal arguments are excluded because they have no host counterpart to
+ // begin with, so a null pointer there does not indicate absence.
+ if (!ArgPtr &&
+ !(ArgType & (TGT_ACC_MAPTYPE_PRIVATE | TGT_ACC_MAPTYPE_LITERAL |
+ TGT_ACC_MAPTYPE_DEVPTR))) {
+ ODBG(ADT_Interface) << "Argument is absent - nothing to map.";
+ if (KI)
+ KI->addArg(nullptr);
+ return;
+ }
+
+ assert(!!ArgBasePtr == !!(ArgType & TGT_ACC_MAPTYPE_PTR_AND_OBJ));
+
+ bool IsNoCreate = ArgType & TGT_ACC_MAPTYPE_NO_CREATE;
+ bool HasPresentModifier = ArgType & TGT_ACC_MAPTYPE_PRESENT;
+ auto AddArg = [&](TargetPointerResultTy &TPR, void *TgtArg, void *HstArg) {
+ if (KI) {
+ if (TPR.isPresent()) {
+ KI->addArg(TgtArg);
+ } else if (HasPresentModifier) {
+ REPORT_FATAL() << Loc
+ << "Device mapping does not exist for present argument";
+ } else {
+ assert(IsNoCreate);
+ KI->addArg(HstArg);
+ }
+ }
+ };
+
+ if (ArgType & TGT_ACC_MAPTYPE_DEVPTR) {
+ if (!KI) {
+ ODBG(ADT_Interface) << "DEVPTR arg in non-kernel context - ignoring.";
+ return;
+ }
+ if (!ArgDesc) {
+ void *LiteralValue = *reinterpret_cast<void ***>(ArgPtr);
+ ODBG(ADT_Interface) << "Got literal device pointer: " << LiteralValue;
+ KI->addArg(LiteralValue);
+ return;
+ }
+ // DEVPTR with a descriptor. The kernel is compiled to receive a device-side
+ // descriptor as a pointer.
+ ODBG(ADT_Interface) << "DEVPTR with descriptor";
+ ArgDescriptorsTy Descs = parseAndVerifyArgDescs(Loc, ArgDesc);
+ Descs.dataBeginDevPtr(Loc, Device, AsyncInfo, HDTTMap, *KI);
+ return;
+ }
+
+ const bool HasFlagTo = ArgType & TGT_ACC_MAPTYPE_TO;
+ if (ArgType & TGT_ACC_MAPTYPE_PRIVATE) {
+ assert(KI && "Private arg should only appear on kernels");
+
+ // A firstprivate copy of an absent optional has no source to read from.
+ if (HasFlagTo && !ArgPtr) {
+ ODBG(ADT_Interface) << "Absent firstprivate argument - passing null.";
+ KI->addArg(nullptr);
+ return;
+ }
+
+ int64_t BaseAllocSize = ArgSize;
+
+ if (BaseAllocSize <= 0)
+ REPORT_FATAL() << "Invalid private variable size";
+
+ int64_t NumPrivate = 1;
+ if ((ArgType & TGT_ACC_MAPTYPE_GANG_PRIVATE))
+ NumPrivate *= KI->KernelArgs.NumGangs[0] * KI->KernelArgs.NumGangs[1] *
+ KI->KernelArgs.NumGangs[2];
+ if ((ArgType & TGT_ACC_MAPTYPE_WORKER_PRIVATE))
+ NumPrivate *= KI->KernelArgs.NumWorkers;
+ if ((ArgType & TGT_ACC_MAPTYPE_VECTOR_PRIVATE))
+ NumPrivate *= KI->KernelArgs.VectorLength;
+
+ if (ArgDesc) {
+ if (NumPrivate != 1)
+ REPORT_FATAL() << Loc << " Multi-dim private array variable is invalid";
+ ODBG() << "Arg desc on private variable";
+ ArgDescriptorsTy Descs = parseAndVerifyArgDescs(Loc, ArgDesc);
+ Descs.dataBeginPrivate(Loc, ArgPtr, ArgSize, HasFlagTo, Device, AsyncInfo,
+ HDTTMap, *KI);
+ } else {
+ void *PrivateMemory = Device.allocData(BaseAllocSize * NumPrivate,
+ nullptr, TARGET_ALLOC_DEFAULT);
+ ODBG(ADT_Interface) << "Allocated private memory with size "
+ << BaseAllocSize << " (" << NumPrivate
+ << " instances) at " << PrivateMemory;
+
+ if (HasFlagTo) {
+ if (NumPrivate != 1)
+ REPORT_FATAL() << Loc
+ << " Multi-dim private variable with copy is invalid";
+
+ assert(ArgPtr);
+ Device.submitData(PrivateMemory, ArgPtr, ArgSize, AsyncInfo,
+ /*Entry=*/nullptr, &HDTTMap);
+ }
+ KI->addArg(PrivateMemory);
+ KI->addLaunchAlloc(PrivateMemory);
+ }
+
+ return;
+ }
+
+ if (ArgType & TGT_ACC_MAPTYPE_LITERAL) {
+ assert(KI && "Literal arg should only appear on kernels");
+ assert(ArgSize && "We need size information to pass in literal args");
+ assert(!ArgDesc);
+ // Our codegen uses indirection for literal args.
+ if (ArgSize <= (int)sizeof(void *)) {
+ // If it is possible to type pun to pointer (i.e. the type width is no
+ // bigger than a pointer, then pass it in literally.
+ void *LiteralValue = *reinterpret_cast<void ***>(ArgPtr);
+ KI->addArg(LiteralValue);
+ return;
+ } else {
+ REPORT_FATAL() << "TODO need to move memory to device";
+ // KI->addArg(DeviceArgPtr);
+ return;
+ }
+ }
+
+ assert(ArgPtr && "We need to have a pointer for data mapping");
+
+ void *ParentAllocation = nullptr;
+ void *DescriptorAddr;
+ bool IsPtrAndObj = ArgType & TGT_ACC_MAPTYPE_PTR_AND_OBJ;
+ if (IsPtrAndObj) {
+ ODBG() << "We got a parent object.";
+ assert(ArgBasePtr);
+ if (Device.getMappingInfo().getTgtPtrBegin(HDTTMap, ArgBasePtr, 1)) {
+ ParentAllocation = ArgBasePtr;
+ DescriptorAddr = ArgBasePtr;
+ } else {
+ // PTR_AND_OBJ but parent not present on device (e.g. enter data copyin
+ // of a pointer component without its parent struct). Data is already
+ // mapped standalone; skip descriptor attach.
+ ODBG() << "Parent not present on device - mapping standalone.";
+ IsPtrAndObj = false;
+ DescriptorAddr = ArgPtr;
+ }
+ } else {
+ DescriptorAddr = ArgPtr;
+ }
+
+ int64_t MappingSize = getMappingSize(ArgSize, ArgType, ArgDesc);
+ if (MappingSize > 0) {
+ ODBG() << "No descriptor parsing needed; mapping size is " << MappingSize;
+
+ if (IsPtrAndObj) {
+ // Map the pointee data, then release entry lock before looking up
+ // the parent for pointer attachment.
+ void *MemTgtPtr = nullptr;
+ {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, ArgPtr, ArgPtr, 0, MappingSize, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ HasPresentModifier,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ AddArg(TPR, TPR.TargetPointer, ArgPtr);
+ if (TPR.isPresent())
+ MemTgtPtr = TPR.TargetPointer;
+ else
+ assert(IsNoCreate);
+ }
+
+ // Update the parent's pointer field on device.
+ if (MemTgtPtr) {
+ LookupResult ParentLR = Device.getMappingInfo().lookupMapping(
+ HDTTMap, DescriptorAddr, sizeof(void *));
+ if (ParentLR.TPR.getEntry()) {
+ void **HstPtrAddr = reinterpret_cast<void **>(DescriptorAddr);
+ uintptr_t TgtDescAddr = ParentLR.TPR.getEntry()->TgtPtrBegin +
+ (reinterpret_cast<uintptr_t>(DescriptorAddr) -
+ ParentLR.TPR.getEntry()->HstPtrBegin);
+ void **TgtPtrAddr = reinterpret_cast<void **>(TgtDescAddr);
+
+ void *HstPteeBase = *HstPtrAddr;
+ void *TgtPteeBase = reinterpret_cast<void *>(
+ reinterpret_cast<uintptr_t>(MemTgtPtr) -
+ (reinterpret_cast<uintptr_t>(ArgPtr) -
+ reinterpret_cast<uintptr_t>(HstPteeBase)));
+
+ if (ParentLR.TPR.getEntry()->addShadowPointer(
+ ShadowPtrInfoTy{HstPtrAddr, TgtPtrAddr, TgtPteeBase,
+ static_cast<int64_t>(sizeof(void *))})) {
+ ODBG() << "PTR_AND_OBJ attach: device field " << TgtPtrAddr
+ << " -> " << TgtPteeBase;
+ void *&Buf = AsyncInfo.getVoidPtrLocation();
+ Buf = TgtPteeBase;
+ Device.submitData(TgtPtrAddr, &Buf, sizeof(void *), AsyncInfo,
+ ParentLR.TPR.getEntry(), &HDTTMap);
+ ParentLR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo);
+ }
+ }
+ }
+
+ } else {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTargetPointer(
+ HDTTMap, ArgPtr, ArgPtr, 0, MappingSize, ArgName, HasFlagTo,
+ /*HasFlagAlways=*/false, /*IsImplicit=*/false,
+ /*UpdateRefCount=*/true, /*HasCloseModifier=*/false,
+ HasPresentModifier,
+ /*HasHoldModifier=*/MapType == AccRefCountingType::Structured,
+ IsNoCreate, AsyncInfo, /*OwnedTPR=*/nullptr,
+ /*ReleaseHDTTMap=*/false);
+ AddArg(TPR, TPR.TargetPointer, ArgPtr);
+ }
+ return;
+ }
+
+ ArgDescriptorsTy Descs = parseAndVerifyArgDescs(Loc, ArgDesc);
+ Descs.dataBegin(Loc, ArgPtr, DescriptorAddr, ParentAllocation, IsPtrAndObj,
+ mapsDescriptor(MapType, IsPtrAndObj, KI), ArgName, HasFlagTo,
+ IsNoCreate, MapType, AsyncInfo, Device, HDTTMap, KI);
+}
+
+bool isPresent(DeviceTy &Device, void *Ptr) {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ Ptr, 1, /*UpdateRefCount=*/false, /*UseHoldRefCount=*/false);
+ return TPR.isPresent();
+}
+
+void accTargetDataEnd(ident_t *Loc, void *ArgBasePtr, void *ArgPtr,
+ int64_t ArgSize, int64_t ArgType, char *ArgName,
+ AccDataDesc *ArgDesc, AccRefCountingType MapType,
+ AsyncInfoTy &AsyncInfo, DeviceTy &Device) {
+ // clang-format off
+ ODBG(ADT_Interface)
+ << "targetDataEnd "
+ << "ArgName=" << getNameFromMapping(ArgName) << ", "
+ << "ArgBasePtr=" << ArgBasePtr << ", "
+ << "ArgPtr=" << ArgPtr << ", "
+ << "ArgSize=" << ArgSize << ", "
+ << "ArgType=" << mapTypeToString(ArgType)
+ << " (" << llvm::format_hex(ArgType, 0) << "), "
+ << "ArgDesc=" << ArgDesc;
+ // clang-format on
+
+ // OpenACC 3.4: `if_present` is only valid on `host_data` and `update`
+ // directives.
+ assert(!(ArgType & TGT_ACC_MAPTYPE_IF_PRESENT));
+
+ // Nothing was mapped for an absent optional or an unallocated allocatable,
+ // so there is nothing to release here either.
+ if (!ArgPtr &&
+ !(ArgType & (TGT_ACC_MAPTYPE_PRIVATE | TGT_ACC_MAPTYPE_LITERAL |
+ TGT_ACC_MAPTYPE_DEVPTR))) {
+ ODBG(ADT_Interface) << "Argument is absent - nothing to release.";
+ return;
+ }
+
+ assert(!!ArgBasePtr == !!(ArgType & TGT_ACC_MAPTYPE_PTR_AND_OBJ));
+
+ // These types are only for kernel launches
+ if ((ArgType & TGT_ACC_MAPTYPE_VECTOR_PRIVATE) ||
+ (ArgType & TGT_ACC_MAPTYPE_GANG_PRIVATE) ||
+ (ArgType & TGT_ACC_MAPTYPE_WORKER_PRIVATE) ||
+ (ArgType & TGT_ACC_MAPTYPE_LITERAL) ||
+ (ArgType & TGT_ACC_MAPTYPE_DEVPTR) ||
+ (ArgType & TGT_ACC_MAPTYPE_PRIVATE)) {
+ ODBG(ADT_Interface) << "Kernel launch argument - ignoring.";
+ return;
+ }
+
+ assert(ArgPtr && "We need to have a pointer for data mapping");
+
+ void *ParentAllocation = nullptr;
+ void *DescriptorAddr;
+ bool IsPtrAndObj = ArgType & TGT_ACC_MAPTYPE_PTR_AND_OBJ;
+ if (IsPtrAndObj) {
+ ODBG(ADT_Mapping) << "We got a parent object.";
+ assert(ArgBasePtr);
+ if (isPresent(Device, ArgBasePtr)) {
+ ParentAllocation = ArgBasePtr;
+ DescriptorAddr = ArgBasePtr;
+ } else {
+ // PTR_AND_OBJ but parent not present on device (e.g. enter data copyin
+ // of a pointer component without its parent struct). Data is already
+ // mapped standalone; skip descriptor attach.
+ ODBG(ADT_Mapping) << "Parent not present on device - mapping standalone.";
+ IsPtrAndObj = false;
+ DescriptorAddr = ArgPtr;
+ }
+ } else {
+ DescriptorAddr = ArgPtr;
+ }
+
+ const bool ForceDelete = ArgType & TGT_ACC_MAPTYPE_FINALIZE;
+ const bool HasFlagFrom = ArgType & TGT_ACC_MAPTYPE_FROM;
+ const bool IsNoCreate = ArgType & TGT_ACC_MAPTYPE_NO_CREATE;
+ AccCopyOutType CopyType = AccCopyOutType::Never;
+ if (HasFlagFrom)
+ CopyType = AccCopyOutType::OnDelete;
+ int64_t MappingSize = getMappingSize(ArgSize, ArgType, ArgDesc);
+ if (MappingSize > 0) {
+ ODBG(ADT_Mapping) << "No descriptor parsing needed; mapping size is "
+ << MappingSize;
+
+ handleSingleDataEnd<int64_t>(Loc, DescriptorAddr, ArgPtr, MappingSize,
+ ForceDelete, IsNoCreate, CopyType, MapType,
+ AsyncInfo, Device);
+ return;
+ }
+
+ ArgDescriptorsTy Descs = parseAndVerifyArgDescs(Loc, ArgDesc);
+ Descs.dataEnd(Loc, ArgPtr, DescriptorAddr, ParentAllocation, ArgType,
+ mapsDescriptor(MapType, IsPtrAndObj, /*KI=*/nullptr),
+ ForceDelete, IsNoCreate, CopyType, MapType, AsyncInfo, Device);
+}
+
+void accTargetDataUpdate(ident_t *Loc, void *ArgBasePtr, void *ArgPtr,
+ int64_t ArgSize, int64_t ArgType, char *ArgName,
+ AccDataDesc *ArgDesc, AsyncInfoTy &AsyncInfo,
+ DeviceTy &Device) {
+ // clang-format off
+ ODBG(ADT_Interface)
+ << "update "
+ << "ArgName=" << getNameFromMapping(ArgName) << ", "
+ << "ArgPtr=" << ArgPtr << ", "
+ << "ArgSize=" << ArgSize << ", "
+ << "ArgType=" << mapTypeToString(ArgType)
+ << " (" << llvm::format_hex(ArgType, 0) << "), "
+ << "ArgDesc=" << ArgDesc;
+ // clang-format on
+
+ if ((ArgType & TGT_ACC_MAPTYPE_LITERAL) ||
+ (ArgType & TGT_ACC_MAPTYPE_PRIVATE) || (ArgType & TGT_ACC_MAPTYPE_DEVPTR))
+ return;
+
+ if (!ArgPtr)
+ return;
+
+ if (ArgSize > 0) {
+ TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+ ArgPtr, ArgSize, /*UpdateRefCount=*/false, /*UseHoldRefCount=*/false,
+ /*MustContain=*/true);
+ if (!TPR.isPresent()) {
+ if (ArgType & TGT_ACC_MAPTYPE_IF_PRESENT) {
+ ODBG(ADT_Interface) << "Not present, if_present - skipping update.";
+ return;
+ }
+ REPORT_FATAL() << "Device mapping does not exist for update at " << Loc;
+ }
+ void *TgtPtr = TPR.TargetPointer;
+ if (ArgType & TGT_ACC_MAPTYPE_TO) {
+ ODBG(ADT_Interface) << "Update TO: " << ArgSize << " bytes hst:" << ArgPtr
+ << " -> tgt:" << TgtPtr;
+ Device.submitData(TgtPtr, ArgPtr, ArgSize, AsyncInfo, TPR.getEntry());
+ }
+ if (ArgType & TGT_ACC_MAPTYPE_FROM) {
+ ODBG(ADT_Interface) << "Update FROM: " << ArgSize
+ << " bytes tgt:" << TgtPtr << " -> hst:" << ArgPtr;
+ Device.retrieveData(ArgPtr, TgtPtr, ArgSize, AsyncInfo, TPR.getEntry());
+ }
+ return;
+ }
+
+ ArgDescriptorsTy Descs = parseAndVerifyArgDescs(Loc, ArgDesc);
+ Descs.dataUpdate(Loc, ArgPtr, ArgType, AsyncInfo, Device);
+}
+
+/// Bring the runtime up on first use. A program whose only OpenACC constructs
+/// are data or `host_data` directives has no device code, so the link produces
+/// no offload binary descriptor and `__tgt_acc_register_lib` never runs, yet
+/// the entry points below still need a device manager.
+static void ensureAccRuntimeInitialized();
+
+template <typename T>
+void withDeviceAndQueue(int64_t DeviceType, int64_t Async, T Callback) {
+ ensureAccRuntimeInitialized();
+
+ llvm::Expected<DeviceTy &> DeviceOrErr =
+ DM->getDevice(static_cast<acc_device_t>(DeviceType));
+ if (!DeviceOrErr)
+ REPORT_FATAL() << "Failed to get device: "
+ << toString(DeviceOrErr.takeError());
+
+ DeviceTy &Device = *DeviceOrErr;
+
+ ODBG(ADT_Interface) << "with device type " << DeviceType << " and async "
+ << asyncToString(Async);
+
+ if (Async == AccAsyncSync) {
+ AsyncInfoTy AsyncInfo(Device);
+ Callback(Device, AsyncInfo);
+ } else {
+ QueueAsyncInfoWrapperTy QueueAsyncInfo(Device, Async);
+ AsyncInfoTy &AsyncInfo = QueueAsyncInfo;
+ Callback(Device, AsyncInfo);
+ }
+}
+
+template <typename FuncTy, typename... ArgsTy>
+void forEachArg(FuncTy Func, bool Increasing, ident_t *Loc, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs, int64_t *ArgSizes,
+ int64_t *ArgTypes, char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs, ArgsTy &&...Args) {
+ assert(!ArgMappers && "we currently do not generate mappers");
+ ODBG(ADT_Interface) << "Got " << ArgNum << " args at " << Loc;
+ int32_t Start = Increasing ? 0 : ArgNum - 1;
+ int32_t End = Increasing ? ArgNum : -1;
+ int32_t Increment = Increasing ? 1 : -1;
+ for (int32_t I = Start; I != End; I += Increment) {
+ ODBG(ADT_Interface) << "Handling arg #" << I;
+ char *Name = ArgNames ? ArgNames[I] : nullptr;
+ Func(Loc, ArgBasePtrs[I], ArgPtrs[I], ArgSizes[I], ArgTypes[I], Name,
+ ArgDescs[I], Args...);
+ }
+}
+} // namespace
+
+namespace llvm::acc::target {
+void *accDataEnter(void *ArgBasePtr, void *ArgPtr, int64_t ArgSize,
+ int64_t ArgType, int64_t Async) {
+ void *Result = nullptr;
+ withDeviceAndQueue(
+ acc_device_default, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ {
+ AccKernelArgsTy KA = {};
+ KernelArgsMappingInfoTy KI{KA, {}, {}, {}};
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo()
+ .HostDataToTargetMap.getExclusiveAccessor();
+ accTargetDataBegin(nullptr, ArgBasePtr, ArgPtr, ArgSize, ArgType,
+ nullptr, nullptr, AccRefCountingType::Dynamic,
+ AsyncInfo, Device, HDTTMap, &KI);
+ assert(KI.Args.size() == 1);
+ Result = KI.Args[0];
+ }
+ dumpTargetPointerMappings(nullptr, Device);
+ });
+ return Result;
+}
+} // namespace llvm::acc::target
+
+EXTERN void __tgt_acc_declare(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs,
+ void **ArgPtrs, int64_t *ArgSizes,
+ int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async, __tgt_bin_desc *Desc) {
+ FUNC_LOGGER(Loc);
+ assert(!Desc);
+
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ {
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo()
+ .HostDataToTargetMap.getExclusiveAccessor();
+ forEachArg(accTargetDataBegin, /*Increasing=*/true, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AccRefCountingType::Structured,
+ AsyncInfo, Device, HDTTMap, /*KI=*/nullptr);
+ }
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+EXTERN void __tgt_acc_mirror_alloc(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs) {
+ FUNC_LOGGER(Loc);
+
+ // Declare action recipes run as part of the host allocation, so there is no
+ // queue to join and the mirrored payload must exist before control returns.
+ withDeviceAndQueue(
+ DeviceType, AccAsyncSync, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ {
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo()
+ .HostDataToTargetMap.getExclusiveAccessor();
+ forEachArg(accTargetDataBegin, /*Increasing=*/true, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AccRefCountingType::Structured,
+ AsyncInfo, Device, HDTTMap, /*KI=*/nullptr);
+ }
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+EXTERN void __tgt_acc_mirror_dealloc(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs) {
+ FUNC_LOGGER(Loc);
+
+ // Runs before the host frees the payload, so the release has to complete
+ // while the host addresses are still valid.
+ withDeviceAndQueue(
+ DeviceType, AccAsyncSync, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ forEachArg(accTargetDataEnd, /*Increasing=*/false, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AccRefCountingType::Structured,
+ AsyncInfo, Device);
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+EXTERN void __tgt_acc_data_update(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs, int64_t Async) {
+ FUNC_LOGGER(Loc);
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ forEachArg(accTargetDataUpdate, /*Increasing=*/true, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AsyncInfo, Device);
+ });
+}
+
+EXTERN void __tgt_acc_copy_present_d2h(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs, void **ArgBufs,
+ int64_t Async) {
+ FUNC_LOGGER(Loc);
+
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
+
+ for (uint32_t I = 0; I < ArgNum; ++I) {
+ assert((!ArgMappers || !ArgMappers[I]) &&
+ "we currently do not generate mappers");
+ void *Buffer = ArgBufs[I];
+ ArgBufs[I] = nullptr;
+ if (!Buffer || !ArgPtrs[I] || ArgSizes[I] < 0)
+ continue;
+
+ void *HostPtr = ArgPtrs[I];
+ if (ArgTypes[I] & TGT_ACC_MAPTYPE_DESCRIPTOR) {
+ HostPtr = ArgBasePtrs[I];
+ } else if (ArgDescs[I]) {
+ ArgDescriptorsTy Descs = parseAndVerifyArgDescs(Loc, ArgDescs[I]);
+ ArgDescriptorsTy::ArgMappingInfosTy MappingInfos =
+ Descs.getMappingInfos(Loc, ArgPtrs[I]);
+ for (ArgDescriptorsTy::ArgMappingInfoTy &MappingInfo :
+ MappingInfos) {
+ if (auto *DescAndMem =
+ std::get_if<DescAndMemMappingInfoTy>(&MappingInfo)) {
+ if (DescAndMem->Memory) {
+ HostPtr = DescAndMem->Memory->RawMemoryPtr;
+ break;
+ }
+ } else if (auto *Memory =
+ std::get_if<MemMappingInfoTy>(&MappingInfo)) {
+ HostPtr = Memory->RawMemoryPtr;
+ break;
+ }
+ }
+ }
+
+ if (!HostPtr)
+ continue;
+
+ LookupResult LR = Device.getMappingInfo().lookupMapping(
+ HDTTMap, HostPtr, ArgSizes[I]);
+ if (!LR.Flags.IsContained)
+ continue;
+
+ HostDataToTargetTy *Entry = LR.TPR.getEntry();
+ void *TargetPtr = reinterpret_cast<void *>(
+ Entry->TgtPtrBegin +
+ (reinterpret_cast<uintptr_t>(HostPtr) - Entry->HstPtrBegin));
+ if (Device.retrieveData(Buffer, TargetPtr, ArgSizes[I], AsyncInfo,
+ Entry, &HDTTMap) != OFFLOAD_SUCCESS)
+ REPORT_FATAL() << Loc << "Device-to-host copy failed";
+
+ ArgBufs[I] = Buffer;
+ }
+ });
+}
+
+EXTERN void __tgt_acc_data_enter(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs, int64_t Async) {
+ FUNC_LOGGER(Loc);
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ {
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo()
+ .HostDataToTargetMap.getExclusiveAccessor();
+ forEachArg(accTargetDataBegin, /*Increasing=*/true, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AccRefCountingType::Dynamic,
+ AsyncInfo, Device, HDTTMap, /*KI=*/nullptr);
+ }
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+EXTERN void __tgt_acc_data_exit(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs,
+ void **ArgPtrs, int64_t *ArgSizes,
+ int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async) {
+ FUNC_LOGGER(Loc);
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ forEachArg(accTargetDataEnd, /*Increasing=*/false, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AccRefCountingType::Dynamic, AsyncInfo,
+ Device);
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+EXTERN void __tgt_acc_data_begin(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, uint32_t ArgNum,
+ void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs, int64_t Async) {
+ FUNC_LOGGER(Loc);
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ {
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo()
+ .HostDataToTargetMap.getExclusiveAccessor();
+ assert(!ArgMappers && "we currently do not generate mappers");
+ ODBG(ADT_Interface) << "Got " << ArgNum << " args at " << Loc;
+ for (uint32_t I = 0; I < ArgNum; ++I) {
+ ODBG(ADT_Interface) << "Handling arg #" << I;
+ char *Name = ArgNames ? ArgNames[I] : nullptr;
+ if (!(ArgTypes[I] & TGT_ACC_MAPTYPE_RETURN_PARAM)) {
+ accTargetDataBegin(Loc, ArgBasePtrs[I], ArgPtrs[I], ArgSizes[I],
+ ArgTypes[I], Name, ArgDescs[I],
+ AccRefCountingType::Structured, AsyncInfo,
+ Device, HDTTMap, /*KI=*/nullptr);
+ continue;
+ }
+
+ // Hand the device address a kernel launch would receive back to
+ // the caller through ArgBasePtrs. The input is the parent object
+ // for PTR_AND_OBJ and null otherwise.
+ AccKernelArgsTy KA = {};
+ KernelArgsMappingInfoTy KI{KA, {}, {}, {}};
+ accTargetDataBegin(Loc, ArgBasePtrs[I], ArgPtrs[I], ArgSizes[I],
+ ArgTypes[I], Name, ArgDescs[I],
+ AccRefCountingType::Structured, AsyncInfo,
+ Device, HDTTMap, &KI);
+ assert(KI.Args.size() == 1);
+ // A no_create argument that is not mapped is returned as the
+ // address it was given, which may already be a device address.
+ void *DevPtr = KI.Args[0];
+ ODBG(ADT_Interface) << "Returning device pointer " << DevPtr;
+ ArgBasePtrs[I] = DevPtr;
+ }
+ }
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+EXTERN void __tgt_acc_data_end(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs,
+ void **ArgPtrs, int64_t *ArgSizes,
+ int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async) {
+ FUNC_LOGGER(Loc);
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ forEachArg(accTargetDataEnd, /*Increasing=*/false, Loc, ArgNum,
+ ArgBasePtrs, ArgPtrs, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers, ArgDescs, AccRefCountingType::Structured,
+ AsyncInfo, Device);
+ dumpTargetPointerMappings(Loc, Device);
+ });
+}
+
+void *getDeviceEntryPtr(void *HostPtr, DeviceTy &Device) {
+ int32_t DeviceId = Device.DeviceID;
+ TableMap *TM = PM->getTableMap(HostPtr);
+ __tgt_target_table *TargetTable = nullptr;
+ {
+ std::lock_guard<std::mutex> TrlTblLock(PM->TrlTblMtx);
+ assert(TM->Table->TargetsTable.size() > (size_t)DeviceId &&
+ "Not expecting a device ID outside the table's bounds!");
+ TargetTable = TM->Table->TargetsTable[DeviceId];
+ }
+ assert(TargetTable && "Global data has not been mapped\n");
+
+ void *TgtEntryPtr = TargetTable->EntriesBegin[TM->Index].Address;
+ ODBG(ADT_Kernel) << "Launching target execution "
+ << TargetTable->EntriesBegin[TM->Index].SymbolName
+ << " with pointer " << TgtEntryPtr << " (index=" << TM->Index
+ << ").";
+ return TgtEntryPtr;
+}
+
+// TODO remove Desc type
+// TODO also remove Mappers argument from everything
+EXTERN int __tgt_acc_kernel(ident_t *Loc, void *Kernel, int64_t Flags,
+ int64_t DeviceType, AccKernelArgsTy *Args,
+ int64_t Async, const char *KernelName,
+ __tgt_bin_desc *Desc) {
+ FUNC_LOGGER(Loc);
+ assert(!Desc);
+
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
+
+ SmallVector<void *> TgtArgs;
+ SmallVector<ptrdiff_t> TgtOffsets;
+ KernelArgsMappingInfoTy KI{*Args, {}, {}, {}};
+ forEachArg(accTargetDataBegin, /*Increasing=*/true, Loc, Args->ArgNum,
+ Args->ArgBasePtrs, Args->ArgPtrs, Args->ArgSizes,
+ Args->ArgTypes, Args->ArgNames, Args->ArgMappers,
+ Args->ArgDescs, AccRefCountingType::Structured, AsyncInfo,
+ Device, HDTTMap, &KI);
+ HDTTMap.destroy();
+
+ KernelLaunchParamsTy LaunchParams = KI.getLaunchArgs();
+ KernelArgsTy DeviceArgs = {0};
+ DeviceArgs.Version = 4;
+ DeviceArgs.ArgPtrs = reinterpret_cast<void **>(&LaunchParams);
+ DeviceArgs.Flags.IsCUDA = true;
+ DeviceArgs.Flags.IsPtrArgs = true;
+ DeviceArgs.Flags.StrictBlocks = true;
+ DeviceArgs.Flags.StrictThreads = true;
+ DeviceArgs.DynCGroupMem = Args->SmemSize;
+
+ DeviceArgs.UserNumBlocks[0] = Args->NumGangs[0];
+ DeviceArgs.UserNumBlocks[1] = Args->NumGangs[1];
+ DeviceArgs.UserNumBlocks[2] = Args->NumGangs[2];
+ DeviceArgs.UserThreadLimit[0] = Args->VectorLength;
+ DeviceArgs.UserThreadLimit[1] = Args->NumWorkers;
+ DeviceArgs.UserThreadLimit[2] = 1;
+
+ void *TgtEntryPtr = getDeviceEntryPtr(Kernel, Device);
+ ODBG(ADT_Interface)
+ << "Launching device kernel " << KernelName
+ << " with entry hst: " << Kernel << " tgt: " << TgtEntryPtr
+ << " with " << KI.Args.size() << " (" << Args->ArgNum << ") args";
+ ODBG(ADT_Interface) << "NumGangs " << Args->NumGangs[0] << ", "
+ << Args->NumGangs[1] << ", " << Args->NumGangs[2];
+ ODBG(ADT_Interface) << "VectorLength " << Args->VectorLength;
+ ODBG(ADT_Interface) << "NumWorkers " << Args->NumWorkers;
+ ODBG(ADT_Interface) << "SmemSize " << Args->SmemSize;
+
+ assert(KI.Args.size() == LaunchParams.NumArgs);
+ for (unsigned I = 0; I < KI.Args.size(); I++)
+ ODBG(ADT_Interface) << "Arg #" << I << ": " << KI.Args[I];
+
+ if (Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(),
+ DeviceArgs, nullptr,
+ AsyncInfo) != OFFLOAD_SUCCESS)
+ REPORT_FATAL() << "Kernel launch failed";
+
+ forEachArg(accTargetDataEnd, /*Increasing=*/false, Loc, Args->ArgNum,
+ Args->ArgBasePtrs, Args->ArgPtrs, Args->ArgSizes,
+ Args->ArgTypes, Args->ArgNames, Args->ArgMappers,
+ Args->ArgDescs, AccRefCountingType::Structured, AsyncInfo,
+ Device);
+
+ dumpTargetPointerMappings(Loc, Device);
+
+ auto LaunchAllocDeleter = [Device = &Device,
+ LaunchAllocs = KI.LaunchAllocs]() {
+ for (void *LaunchAlloc : LaunchAllocs)
+ if (int32_t Ret = Device->deleteData(LaunchAlloc);
+ Ret != OFFLOAD_SUCCESS)
+ return Ret;
+ return OFFLOAD_SUCCESS;
+ };
+ AsyncInfo.addPostProcessingFunction(LaunchAllocDeleter);
+ });
+ return OFFLOAD_SUCCESS;
+}
+
+EXTERN void *__tgt_acc_get_deviceptr(ident_t *Loc, void *BasePtr, int64_t Flags,
+ void *HostPtr) {
+ FUNC_LOGGER(Loc);
+ ODBG(ADT_Interface) << Loc << "BasePtr: " << BasePtr << ", "
+ << "Flags: " << llvm::format_hex(Flags, 0) << ", "
+ << "HostPtr: " << HostPtr;
+
+ void *DevicePtr = nullptr;
+
+ ensureAccRuntimeInitialized();
+
+ llvm::Expected<DeviceTy &> DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << "Failed to get device: "
+ << toString(DeviceOrErr.takeError());
+ DeviceTy &Device = *DeviceOrErr;
+
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
+ DevicePtr = Device.getMappingInfo().getTgtPtrBegin(HDTTMap, HostPtr, 0);
+
+ ODBG(ADT_Interface) << "DevicePtr: " << DevicePtr;
+
+ return DevicePtr;
+}
+
+EXTERN void __tgt_acc_set_default_async(ident_t *Loc, int64_t Async) {
+ FUNC_LOGGER(Loc);
+ ODBG(ADT_Interface) << Loc << ": Set async=" << asyncToString(Async);
+
+ if (Async == AccAsyncSync) {
+ REPORT_FATAL() << Loc
+ << "The default queue cannot be set to `acc_async_sync'";
+ } else if (Async == AccAsyncNoval) {
+ REPORT_FATAL() << Loc
+ << "The default queue cannot be set to `acc_async_noval'";
+ } else if (Async == AccAsyncDefault) {
+ Async = AccAsyncDefaultQueue;
+ } else if (Async < 0) {
+ REPORT_FATAL() << Loc << "Negative queues are invalid";
+ }
+
+ icv::AccDefaultAsyncVar = Async;
+}
+
+EXTERN void __tgt_acc_set_device_num(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType, int64_t DeviceNum) {
+ FUNC_LOGGER(Loc);
+ // OpenACC 3.3: If the value of device_num argument is negative, the runtime
+ // will revert to the default behavior, which is implementation-defined. A set
+ // device_num directive is functionally equivalent
+ if (DeviceNum < 0) {
+ DeviceNum = 0;
+ }
+
+ // OpenACC 3.3: If the value of the device_type argument is zero or the clause
+ // does not appear, the selected device number will be used for all attached
+ // accelerator types.
+ if (DeviceType == 0) {
+ DM->setAllDeviceId(DeviceNum);
+ return;
+ }
+
+ DM->setDeviceType(static_cast<acc_device_t>(DeviceType));
+ DM->setDeviceId(DeviceNum);
+}
+
+EXTERN void __tgt_acc_set_device_type(ident_t *Loc, int64_t Flags,
+ int64_t DeviceType) {
+ FUNC_LOGGER(Loc);
+ DM->setDeviceType(static_cast<acc_device_t>(DeviceType));
+}
+
+EXTERN int __tgt_acc_wait(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int32_t DeviceNum, uint32_t WaitNum,
+ int64_t *WaitList, int64_t Async) {
+ FUNC_LOGGER(Loc);
+ ODBG(ADT_Interface) << Loc << "\n"
+ << "DeviceNum: " << DeviceNum << ", "
+ << "DeviceType: " << DeviceType << ", "
+ << "WaitNum: " << WaitNum;
+ for (size_t I = 0; I < WaitNum; I++) {
+ ODBG(ADT_Interface) << "WaitList[" << I
+ << "]: " << asyncToString(WaitList[I]);
+ }
+ ODBG(ADT_Interface) << "Async: " << asyncToString(Async)
+ << " Flags: " << llvm::format_hex(Flags, 0);
+
+ ensureAccRuntimeInitialized();
+
+ accAsyncWait(Loc, DM->getPMDeviceId(), WaitNum, WaitList);
+
+ return 0;
+}
+
+namespace {
+static std::mutex InitMutex;
+} // namespace
+
+namespace {
+void ensureAccRuntimeInitialized() {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ if (llvm::acc::target::DM)
+ return;
+
+ FUNC_LOGGER();
+ initAccRuntime();
+ std::atexit([]() {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ FUNC_LOGGER();
+ deinitAccRuntime();
+ });
+}
+} // namespace
+
+EXTERN void __tgt_acc_rtl_init() { initAccRuntime(); }
+
+EXTERN void __tgt_acc_rtl_deinit() { deinitAccRuntime(); }
+
+EXTERN void __tgt_acc_register_rpc_callback(unsigned (*Callback)(void *,
+ unsigned)) {
+ llvm::omp::target::helpers::registerRPCCallback(Callback);
+}
+
+/// Sentinel referenced by objects compiled with OpenACC. Defining it here makes
+/// linking succeed only when the OpenACC runtime is part of the link.
+EXTERN void __acc_compiled() {}
+
+EXTERN void __tgt_acc_init(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int64_t DeviceNum) {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ FUNC_LOGGER(Loc);
+ REPORT_WARN() << "acc init ignores user's request and initializes all "
+ "available devices.";
+ initAccRuntime();
+ std::atexit([]() {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ FUNC_LOGGER();
+ deinitAccRuntime();
+ });
+}
+
+EXTERN void __tgt_acc_shutdown(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int64_t DeviceNum) {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ FUNC_LOGGER(Loc);
+ REPORT_WARN() << "acc shutdown is ignored.";
+}
+
+EXTERN void __tgt_acc_register_lib(__tgt_bin_desc *Desc) {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ FUNC_LOGGER();
+ initAccRuntime();
+ if (PM->delayRegisterLib(__tgt_acc_register_lib, Desc))
+ return;
+
+ PM->registerLib(Desc);
+ llvm::acc::target::DM->refreshDeviceMapping(/*UpdateDeviceType=*/true);
+}
+
+EXTERN void __tgt_acc_unregister_lib(__tgt_bin_desc *Desc) {
+ std::scoped_lock<decltype(InitMutex)> Lock(InitMutex);
+ FUNC_LOGGER();
+ PM->unregisterLib(Desc);
+
+ deinitAccRuntime();
+}
+
+EXTERN void *__tgt_acc_alloc(ident_t *Loc, int64_t DeviceType, int64_t Size,
+ int64_t Async) {
+ FUNC_LOGGER(Loc);
+ ODBG(ADT_Interface) << "Allocating " << Size << " bytes";
+ void *Rc = nullptr;
+ withDeviceAndQueue(DeviceType, Async,
+ [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ Rc = Device.allocData(Size, nullptr);
+ });
+ ODBG(ADT_Interface) << "Device ptr " << Rc;
+ return Rc;
+}
+
+EXTERN void __tgt_acc_free(ident_t *Loc, int64_t DeviceType, void *Ptr,
+ int64_t Async) {
+ FUNC_LOGGER(Loc);
+ ODBG(ADT_Interface) << "Address " << Ptr;
+ withDeviceAndQueue(
+ DeviceType, Async, [&](DeviceTy &Device, AsyncInfoTy &AsyncInfo) {
+ if (Device.deleteData(Ptr) == OFFLOAD_FAIL)
+ REPORT_FATAL() << "Failed to deallocate device ptr. Set "
+ "OFFLOAD_TRACK_ALLOCATION_TRACES=1 to track "
+ "allocations.";
+ });
+}
diff --git a/offload/libacctarget/Interface.h b/offload/libacctarget/Interface.h
new file mode 100644
index 00000000000000..6c8068b3e6d4e0
--- /dev/null
+++ b/offload/libacctarget/Interface.h
@@ -0,0 +1,301 @@
+//===- Interface.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __TGT_ACC_TARGET_H__
+#define __TGT_ACC_TARGET_H__
+
+#ifdef __cplusplus
+#include "flang-rt/runtime/descriptor.h"
+using namespace Fortran::ISO;
+#else
+#include "flang/ISO_Fortran_binding.h"
+#endif
+
+#include "Shared/APITypes.h"
+#include "Shared/SourceInfo.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Portable alignment attribute for C89/C99 compatibility
+#if defined(_MSC_VER)
+#define ACC_ALIGNED(x) __declspec(align(x))
+#elif defined(__GNUC__) || defined(__clang__)
+#define ACC_ALIGNED(x) __attribute__((aligned(x)))
+#else
+#define ACC_ALIGNED(x)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//===----------------------------------------------------------------------===//
+// OpenACC Target Offload Types and Enums
+//===----------------------------------------------------------------------===//
+
+enum {
+ // no flags
+ TGT_ACC_MAPTYPE_NONE = 0x0,
+ // copy data from host to device
+ TGT_ACC_MAPTYPE_TO = 0x1, // enter data
+ // copy data from device to host
+ TGT_ACC_MAPTYPE_FROM = 0x2, // exit data
+ // force unmapping of data
+ TGT_ACC_MAPTYPE_FINALIZE = 0x8,
+ // map the pointer as well as the pointee
+ TGT_ACC_MAPTYPE_PTR_AND_OBJ = 0x10,
+ // return the device address of the mapped data through ArgBasePtrs (input:
+ // parent for PTR_AND_OBJ, else null); pass the original base to data_end
+ TGT_ACC_MAPTYPE_RETURN_PARAM = 0x40,
+ // private variable - not mapped
+ TGT_ACC_MAPTYPE_PRIVATE = 0x80,
+ // copy by value - not mapped
+ TGT_ACC_MAPTYPE_LITERAL = 0x100,
+ // implicitly determined data attribute
+ TGT_ACC_MAPTYPE_IMPLICIT = 0x200,
+ // device pointer - already mapped
+ TGT_ACC_MAPTYPE_DEVPTR = 0x400,
+ // device pointer
+ TGT_ACC_MAPTYPE_MANAGED_DEVPTR = 0x800,
+ // present or don't create
+ TGT_ACC_MAPTYPE_NO_CREATE = 0x2000,
+ // private variable - gang
+ TGT_ACC_MAPTYPE_GANG_PRIVATE = 0x4000,
+ // private variable - worker
+ TGT_ACC_MAPTYPE_WORKER_PRIVATE = 0x8000,
+ // private variable - vector
+ TGT_ACC_MAPTYPE_VECTOR_PRIVATE = 0x10000,
+ // zero modifier
+ TGT_ACC_MAPTYPE_INIT_ZERO = 0x20000,
+ // device resident memory - persistent allocation
+ TGT_ACC_MAPTYPE_DEVICE_RESIDENT = 0x40000,
+ // present or not
+ TGT_ACC_MAPTYPE_IF_PRESENT = 0x80000,
+ // require an existing mapping
+ TGT_ACC_MAPTYPE_PRESENT = 0x100000,
+ // operate on the descriptor rather than the data it describes
+ TGT_ACC_MAPTYPE_DESCRIPTOR = 0x200000,
+ // reduction variable
+ TGT_ACC_MAPTYPE_REDUCTION = 0x400000,
+};
+
+/// Array descriptor types
+enum {
+ TGT_ACC_DESC_GENERIC = 0, // Generic type descriptor.
+ TGT_ACC_DESC_F18 = 1, // Fortran 2018 type descriptor.
+ TGT_ACC_DESC_MEMREF = 2, // MemRef type descriptor.
+ TGT_ACC_DESC_OPENACC = 0x1000 // OpenACC descriptor.
+};
+
+/// Device pointer type.
+typedef uintptr_t tgt_acc_devptr_t;
+
+/// Type descriptor base struct.
+typedef struct {
+ // Version of the descriptor.
+ int32_t Version;
+} AccDataDesc;
+
+/// Generic type descriptor.
+typedef struct {
+ AccDataDesc Base;
+} AccDataDescGeneric;
+
+/// F18 type descriptor.
+typedef struct {
+ AccDataDesc Base;
+ CFI_cdesc_t *FortranDescriptor;
+} AccDataDescF18;
+
+/// The structure defined by LLVMTypeConverter::getMemRefDescriptorFields.
+typedef struct {
+ void *allocatedPtr;
+ void *alignedPtr;
+ uint64_t offset;
+
+ uint64_t sizes[1];
+// Below are the real fields in the struct where Rank is a compile-time
+// constant. We use offsets from the above sizes to obtain the addresses of
+// the sizes and strides arrays.
+#if 0
+ uint64_t sizes[Rank];
+ uint64_t strides[Rank];
+#endif
+} MemRefDesc;
+
+/// MemRef type descriptor.
+typedef struct {
+ AccDataDesc Base;
+ unsigned char Rank;
+ uint64_t ElementSize;
+ MemRefDesc *MemRefDescriptor;
+} AccDataDescMemRef;
+
+/// OpenACC descriptor.
+typedef struct {
+ AccDataDesc Base;
+ ACC_ALIGNED(8) unsigned char Rank;
+ int64_t ElementSize;
+ int64_t *LowerBounds;
+ int64_t *UpperBounds;
+ int64_t *Extents;
+ int64_t *StridesInBytes;
+ int64_t *StartIndices;
+} AccDataDescOpenACC;
+
+/// This struct contains all of the arguments to a target kernel region launch.
+typedef struct {
+ // Version of this struct for ABI compatibility.
+ uint32_t Version;
+ // Number of arguments in each input pointer.
+ uint32_t ArgNum;
+ // Base pointer of each argument (e.g. a struct).
+ void **ArgBasePtrs;
+ // Pointer to the argument data.
+ void **ArgPtrs;
+ // Size of the argument data in bytes.
+ int64_t *ArgSizes;
+ // Type of the data (e.g. to / from).
+ int64_t *ArgTypes;
+ // Name of the data for debugging, possibly null.
+ char **ArgNames;
+ // User-defined mappers (e.g. C++ copy ctors), possibly null.
+ void **ArgMappers;
+ // Type descriptors.
+ AccDataDesc **ArgDescs;
+ // Loop tripcount.
+ uint64_t Tripcount;
+ // Values of the num_gangs clause, in three dimensions.
+ int64_t NumGangs[3];
+ // Value of the num_workers clause.
+ int64_t NumWorkers;
+ // Value of the vector_length clause.
+ int64_t VectorLength;
+ // Size of shared memory.
+ int64_t SmemSize;
+} AccKernelArgsTy;
+
+//===----------------------------------------------------------------------===//
+// OpenACC Target Offload Runtime Compiler Interface API
+//===----------------------------------------------------------------------===//
+
+/// adds a target shared library to the target execution image
+void __tgt_acc_register_lib(__tgt_bin_desc *Desc);
+
+/// removes a target shared library from the target execution image
+void __tgt_acc_unregister_lib(__tgt_bin_desc *Desc);
+
+/// 'acc init' directive
+void __tgt_acc_init(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int64_t DeviceNum);
+
+/// 'acc shutdown' directive
+void __tgt_acc_shutdown(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int64_t DeviceNum);
+
+/// 'acc declare' directive
+void __tgt_acc_declare(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs, int64_t Async,
+ __tgt_bin_desc *Desc);
+
+/// 'acc enter data' directive
+void __tgt_acc_data_enter(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async);
+
+/// 'acc exit data' directive
+void __tgt_acc_data_exit(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async);
+
+/// 'acc update' directive
+void __tgt_acc_data_update(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs, int64_t Async);
+
+/// Copy present device values into caller-provided host buffers.
+void __tgt_acc_copy_present_d2h(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs,
+ void **ArgPtrs, int64_t *ArgSizes,
+ int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ void **ArgBufs, int64_t Async);
+
+/// data mapping begin (for `acc data` construct or compute construct)
+void __tgt_acc_data_begin(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async);
+
+/// data mapping end (for `acc data` construct or compute construct)
+void __tgt_acc_data_end(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs,
+ int64_t Async);
+
+/// compute construct directive
+int __tgt_acc_kernel(ident_t *Loc, void *Kernel, int64_t Flags,
+ int64_t DeviceType, AccKernelArgsTy *Args, int64_t Async,
+ const char *KernelName, __tgt_bin_desc *Desc);
+
+/// 'acc wait' directive
+int __tgt_acc_wait(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int32_t DeviceNum, uint32_t WaitNum, int64_t *WaitList,
+ int64_t Async);
+
+/// Allocate a device buffer that is not tracked by the mapping tables
+void *__tgt_acc_alloc(ident_t *Loc, int64_t DeviceType, int64_t Size,
+ int64_t Async);
+
+/// Deallocate a buffer obtained from __tgt_acc_alloc
+void __tgt_acc_free(ident_t *Loc, int64_t DeviceType, void *Ptr, int64_t Async);
+
+/// `acc host_data use_device` directive
+void *__tgt_acc_get_deviceptr(ident_t *Loc, void *BasePtr, int64_t Flags,
+ void *HostPtr);
+
+/// 'acc set default_async' directive
+void __tgt_acc_set_default_async(ident_t *Loc, int64_t Async);
+
+/// 'acc set device_num' directive
+void __tgt_acc_set_device_num(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ int64_t DeviceNum);
+
+/// 'acc set device_type' directive
+void __tgt_acc_set_device_type(ident_t *Loc, int64_t Flags, int64_t DeviceType);
+
+/// Mirror allocation for declare action recipes
+void __tgt_acc_mirror_alloc(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs, void **ArgPtrs,
+ int64_t *ArgSizes, int64_t *ArgTypes,
+ char **ArgNames, void **ArgMappers,
+ AccDataDesc **ArgDescs);
+
+/// Mirror deallocation for declare action recipes
+void __tgt_acc_mirror_dealloc(ident_t *Loc, int64_t Flags, int64_t DeviceType,
+ uint32_t ArgNum, void **ArgBasePtrs,
+ void **ArgPtrs, int64_t *ArgSizes,
+ int64_t *ArgTypes, char **ArgNames,
+ void **ArgMappers, AccDataDesc **ArgDescs);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // __TGT_ACC_TARGET_H__
diff --git a/offload/libacctarget/Logger.h b/offload/libacctarget/Logger.h
new file mode 100644
index 00000000000000..7ffc9ae8e2bede
--- /dev/null
+++ b/offload/libacctarget/Logger.h
@@ -0,0 +1,47 @@
+//===- Logger.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ACC_OFFLOAD_LOGGER_H_
+#define LLVM_ACC_OFFLOAD_LOGGER_H_
+
+#include "Shared/Debug.h"
+#include "Shared/SourceInfo.h"
+#include <optional>
+
+namespace llvm::acc::target::debug {
+inline std::string formatLoc(ident_t *Loc) {
+ SourceInfo SI(Loc);
+ return std::string(SI.getFilename()) + ":" + std::to_string(SI.getLine());
+}
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, ident_t *Loc) {
+ return OS << formatLoc(Loc);
+}
+struct ScopeLoggerTy {
+ const char *ScopeName;
+ std::optional<ident_t *> Loc = std::nullopt;
+ ScopeLoggerTy(const char *ScopeName, ident_t *Loc)
+ : ScopeName(ScopeName), Loc(Loc) {
+ ODBG() << "> " << ScopeName << "(" << Loc << ")";
+ }
+ ScopeLoggerTy(const char *ScopeName) : ScopeName(ScopeName) {
+ ODBG() << "> " << ScopeName;
+ }
+ ~ScopeLoggerTy() {
+ if (Loc)
+ ODBG() << "< " << ScopeName << "(" << *Loc << ")";
+ else
+ ODBG() << "< " << ScopeName;
+ }
+};
+} // namespace llvm::acc::target::debug
+
+#define FUNC_LOGGER(...) \
+ ScopeLoggerTy FunctionScopeLogger(__FUNCTION__, ##__VA_ARGS__)
+
+#endif // LLVM_ACC_OFFLOAD_LOGGER_H_
diff --git a/offload/libacctarget/Private.h b/offload/libacctarget/Private.h
new file mode 100644
index 00000000000000..c7d15aa23c1899
--- /dev/null
+++ b/offload/libacctarget/Private.h
@@ -0,0 +1,29 @@
+//===- Private.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _OPENACC_PRIVATE_H_
+#define _OPENACC_PRIVATE_H_
+
+#include <cstddef>
+#include <cstdint>
+
+namespace llvm::acc::target {
+int accIsPresent(void *);
+void *accAlloc(size_t);
+void accFree(void *);
+void accMemcpyFromDevice(void *, void *, size_t);
+void accMemcpyToDevice(void *, void *, size_t);
+void accMemcpyD2D(void *, void *, size_t, int, int);
+void accMapData(void *, void *, size_t);
+void accUnmapData(void *);
+
+void *accDataEnter(void *ArgBasePtr, void *ArgPtr, int64_t ArgSize,
+ int64_t ArgType, int64_t Async);
+} // namespace llvm::acc::target
+
+#endif // _OPENACC_PRIVATE_H_
diff --git a/offload/libacctarget/QueueManager.cpp b/offload/libacctarget/QueueManager.cpp
new file mode 100644
index 00000000000000..23f4090fc8bb4d
--- /dev/null
+++ b/offload/libacctarget/QueueManager.cpp
@@ -0,0 +1,180 @@
+//===- QueueManager.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "QueueManager.h"
+#include "PluginManager.h"
+#include "Private.h"
+#include "RuntimeState.h"
+#include "Shared/Debug.h"
+
+namespace llvm::acc::target {
+QueueManagerTy *QueueManager = nullptr;
+} // namespace llvm::acc::target
+
+using namespace llvm::acc::target;
+
+static void synchronizeQueueOrDie(QueueIdTy Queue, DeviceTy &Device,
+ AsyncInfoTy *AsyncInfo) {
+
+ ODBG() << "Synchronizing stream " << AsyncInfo << " for device "
+ << Device.DeviceID << " " << &Device << " with queue ID " << Queue;
+ auto Res = AsyncInfo->synchronize();
+ if (Res != OFFLOAD_SUCCESS)
+ FATAL_MESSAGE(Device.DeviceID,
+ "Failed to synchronize queue %" PRIi32 " on device %d", Queue,
+ Device.DeviceID);
+}
+
+static QueueManagerTy::StatusTy
+queryQueueOrDie(QueueIdTy Queue, DeviceTy &Device, AsyncInfoTy *AsyncInfo) {
+
+ ODBG() << "Querying stream " << AsyncInfo << " for device " << Device.DeviceID
+ << " " << &Device << " with queue ID " << Queue;
+ auto Res = AsyncInfo->query();
+ if (Res == OFFLOAD_FAIL)
+ FATAL_MESSAGE(Device.DeviceID,
+ "Failed to query queue %" PRIi32 " on device %d", Queue,
+ Device.DeviceID);
+ return static_cast<QueueManagerTy::StatusTy>(Res);
+}
+
+QueueManagerTy::QueueManagerTy() {}
+
+QueueManagerTy::~QueueManagerTy() {}
+
+void QueueManagerTy::synchronize() {
+ for (auto &[Key, Q] : QueueMap) {
+ auto &[D, Id] = Key;
+ synchronizeQueueOrDie(Id, *D, Q.get());
+ }
+}
+
+void QueueManagerTy::synchronize(DeviceTy &Device) {
+ for (auto &[Key, Q] : QueueMap) {
+ auto &[D, Id] = Key;
+ if (&Device != D)
+ continue;
+ synchronizeQueueOrDie(Id, Device, Q.get());
+ }
+}
+
+void QueueManagerTy::synchronize(DeviceTy &Device, QueueIdTy Queue) {
+ AsyncInfoTy *AsyncInfo = QueueManager->get(Device, Queue);
+ synchronizeQueueOrDie(Queue, Device, AsyncInfo);
+}
+
+QueueManagerTy::StatusTy QueueManagerTy::query() {
+ for (auto &[Key, Q] : QueueMap) {
+ auto &[D, Id] = Key;
+ if (queryQueueOrDie(Id, *D, Q.get()) == StatusTy::NOT_READY)
+ return StatusTy::NOT_READY;
+ }
+ return StatusTy::READY;
+}
+
+QueueManagerTy::StatusTy QueueManagerTy::query(DeviceTy &Device) {
+ for (auto &[Key, Q] : QueueMap) {
+ auto &[D, Id] = Key;
+ if (&Device != D)
+ continue;
+ if (queryQueueOrDie(Id, *D, Q.get()) == StatusTy::NOT_READY)
+ return StatusTy::NOT_READY;
+ }
+ return StatusTy::READY;
+}
+
+QueueManagerTy::StatusTy QueueManagerTy::query(DeviceTy &Device,
+ QueueIdTy Queue) {
+ AsyncInfoTy *AsyncInfo = QueueManager->get(Device, Queue);
+ return queryQueueOrDie(Queue, Device, AsyncInfo);
+}
+
+AsyncInfoTy *QueueManagerTy::get(DeviceTy &Device, QueueIdTy QueueId) {
+ static std::mutex Mutex;
+ std::lock_guard<std::mutex> G(Mutex);
+
+ auto Insertion = QueueMap.insert({std::make_pair(&Device, QueueId), nullptr});
+ if (Insertion.second) {
+ Insertion.first->second = std::make_unique<AsyncInfoTy>(
+ Device, AsyncInfoTy::SyncTy::STATIC_NON_BLOCKING);
+ ODBG() << "Initialized new stream for device " << &Device << " id "
+ << QueueId << " -> " << Insertion.first->second.get();
+ }
+ return Insertion.first->second.get();
+}
+
+namespace llvm::acc::target {
+void accAsyncWait(ident_t *Loc, int64_t DeviceId, int64_t WaitArg) {
+ int64_t WaitArgs[] = {WaitArg};
+ accAsyncWait(Loc, DeviceId, 1, WaitArgs);
+}
+
+void accAsyncWait(ident_t *Loc, int64_t DeviceId, uint32_t WaitNum,
+ int64_t *WaitList) {
+ ODBG() << "Synchronizing streams for device " << DeviceId;
+
+ auto DeviceOrErr = PM->getDevice(DeviceId);
+ if (!DeviceOrErr)
+ FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
+
+ if (WaitNum == 0) {
+ QueueManager->synchronize(*DeviceOrErr);
+ } else {
+ for (unsigned I = 0; I < WaitNum; I++)
+ QueueManager->synchronize(*DeviceOrErr, WaitList[I]);
+ }
+}
+
+void accAsyncWaitAll(ident_t *Loc, int64_t DeviceId) {
+ ODBG() << "Synchronizing all streams for device " << DeviceId;
+
+ auto DeviceOrErr = PM->getDevice(DeviceId);
+ if (!DeviceOrErr)
+ FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
+ QueueManager->synchronize(*DeviceOrErr);
+}
+
+void accAsyncWaitAll(ident_t *Loc) {
+ ODBG() << "Synchronizing all streams";
+ QueueManager->synchronize();
+}
+
+int accAsyncTest(ident_t *Loc, int64_t DeviceId, int64_t TestArg) {
+ int64_t TestList[] = {TestArg};
+ return accAsyncTest(Loc, DeviceId, 1, TestList);
+}
+
+int accAsyncTest(ident_t *Loc, int64_t DeviceId, uint32_t TestNum,
+ int64_t *TestList) {
+ ODBG() << "Querying streams for device " << DeviceId;
+
+ auto DeviceOrErr = PM->getDevice(DeviceId);
+ if (!DeviceOrErr)
+ FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
+
+ for (unsigned I = 0; I < TestNum; I++)
+ if (QueueManager->query(*DeviceOrErr, TestList[I]) ==
+ QueueManagerTy::StatusTy::NOT_READY)
+ return static_cast<int>(QueueManagerTy::StatusTy::NOT_READY);
+ return static_cast<int>(QueueManagerTy::StatusTy::READY);
+}
+
+int accAsyncTestAll(ident_t *Loc, int64_t DeviceId) {
+ ODBG() << "Querying all streams for device " << DeviceId;
+
+ auto DeviceOrErr = PM->getDevice(DeviceId);
+ if (!DeviceOrErr)
+ FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
+ return static_cast<int>(QueueManager->query(*DeviceOrErr));
+}
+
+int accAsyncTestAll(ident_t *Loc) {
+ ODBG() << "Querying all streams";
+ return static_cast<int>(QueueManager->query());
+}
+} // namespace llvm::acc::target
diff --git a/offload/libacctarget/QueueManager.h b/offload/libacctarget/QueueManager.h
new file mode 100644
index 00000000000000..c1cbefb528b7ab
--- /dev/null
+++ b/offload/libacctarget/QueueManager.h
@@ -0,0 +1,80 @@
+//===- QueueManager.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _QUEUEMANAGER_H_
+#define _QUEUEMANAGER_H_
+
+#include "omptarget.h"
+
+#include <map>
+#include <memory>
+
+namespace llvm::acc::target {
+
+using QueueIdTy = uint32_t;
+
+class QueueManagerTy {
+public:
+ QueueManagerTy();
+ ~QueueManagerTy();
+
+ void init() {}
+ void deinit() {}
+
+ enum class StatusTy { READY = 0, NOT_READY = 1 };
+
+ AsyncInfoTy *get(DeviceTy &Device, QueueIdTy QueueId);
+
+ void synchronize(DeviceTy &Device, QueueIdTy Queue);
+ void synchronize(DeviceTy &Device);
+ void synchronize();
+
+ StatusTy query(DeviceTy &Device, QueueIdTy Queue);
+ StatusTy query(DeviceTy &Device);
+ StatusTy query();
+
+private:
+ std::map<std::pair<DeviceTy *, QueueIdTy>, std::unique_ptr<AsyncInfoTy>>
+ QueueMap;
+};
+
+extern QueueManagerTy *QueueManager;
+
+class QueueAsyncInfoWrapperTy {
+ AsyncInfoTy *AsyncInfo;
+
+public:
+ QueueAsyncInfoWrapperTy(DeviceTy &Device, QueueIdTy QueueId) {
+ AsyncInfo = QueueManager->get(Device, QueueId);
+ }
+
+ ~QueueAsyncInfoWrapperTy() {}
+
+ operator AsyncInfoTy &() { return *AsyncInfo; }
+};
+
+extern QueueManagerTy *QueueManager;
+
+void accAsyncWait(ident_t *Loc, int64_t DeviceId, int64_t WaitArg);
+void accAsyncWait(ident_t *Loc, int64_t DeviceId, uint32_t WaitNum,
+ int64_t *WaitList);
+void accAsyncWaitAll(ident_t *Loc, int64_t DeviceId);
+void accAsyncWaitAll(ident_t *Loc);
+int accAsyncTest(ident_t *Loc, int64_t DeviceId, int64_t TestArg);
+int accAsyncTest(ident_t *Loc, int64_t DeviceId, uint32_t TestNum,
+ int64_t *TestList);
+int accAsyncTestAll(ident_t *Loc, int64_t DeviceId);
+int accAsyncTestAll(ident_t *Loc);
+} // namespace llvm::acc::target
+
+namespace llvm::acc::target::icv {
+// acc-default-async-var
+extern thread_local int32_t AccDefaultAsyncVar;
+} // namespace llvm::acc::target::icv
+
+#endif // _QUEUEMANAGER_H_
diff --git a/offload/libacctarget/RuntimeImpl.cpp b/offload/libacctarget/RuntimeImpl.cpp
new file mode 100644
index 00000000000000..fa645c29fa713d
--- /dev/null
+++ b/offload/libacctarget/RuntimeImpl.cpp
@@ -0,0 +1,175 @@
+//===- AccImpl.cpp ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Debug.h"
+#include "DeviceManager.h"
+#include "Logger.h"
+#include "PluginManager.h"
+#include "Private.h"
+#include "Shared/Debug.h"
+
+using namespace llvm::acc::target::debug;
+
+namespace llvm::acc::target {
+int accIsPresent(void *Ptr) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << "Address " << Ptr;
+
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+
+ TargetPointerResultTy TPR =
+ DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+ /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
+ int Rc = TPR.isPresent();
+ ODBG(ADT_Interface) << "Result " << Rc;
+ return Rc;
+}
+
+void *accAlloc(size_t Size) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << "Allocating " << Size << " bytes";
+
+ if (Size <= 0) {
+ ODBG(ADT_Interface) << "Non-positive length";
+ return NULL;
+ }
+
+ void *Rc = NULL;
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+
+ Rc = DeviceOrErr->allocData(Size, nullptr);
+ ODBG(ADT_Interface) << "Device ptr " << Rc;
+ return Rc;
+}
+
+void accFree(void *DevicePtr) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << "Address " << DevicePtr;
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+
+ if (DeviceOrErr->deleteData(DevicePtr) == OFFLOAD_FAIL)
+ REPORT_FATAL() << "Failed to deallocate device ptr. Set "
+ "OFFLOAD_TRACK_ALLOCATION_TRACES=1 to track allocations.";
+}
+
+void accMemcpyToDevice(void *Dst, void *Src, size_t Bytes) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << Dst << " <- " << Src << ", " << Bytes << " bytes";
+
+ if (!Dst || !Src || Bytes <= 0) {
+ if (Bytes == 0) {
+ ODBG(ADT_Interface) << "Zero bytes, nothing to do";
+ return;
+ }
+ REPORT() << "Invalid arguments";
+ return;
+ }
+
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+ AsyncInfoTy AsyncInfo(*DeviceOrErr);
+ int Rc = DeviceOrErr->submitData(Dst, Src, Bytes, AsyncInfo);
+ ODBG(ADT_Interface) << "Result " << Rc;
+}
+
+void accMemcpyFromDevice(void *Dst, void *Src, size_t Bytes) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << Dst << " <- " << Src << ", " << Bytes << " bytes";
+
+ if (!Dst || !Src || Bytes <= 0) {
+ if (Bytes == 0) {
+ ODBG(ADT_Interface) << "Zero bytes, nothing to do";
+ return;
+ }
+ REPORT() << "Invalid arguments";
+ return;
+ }
+
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+ AsyncInfoTy AsyncInfo(*DeviceOrErr);
+ int Rc = DeviceOrErr->retrieveData(Dst, Src, Bytes, AsyncInfo);
+ ODBG(ADT_Interface) << "Result " << Rc;
+}
+
+void accMemcpyD2D(void *Dst, void *Src, size_t Bytes, int DstDevice,
+ int SrcDevice) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << Dst << " <- " << Src << ", " << Bytes << " bytes";
+
+ if (!Dst || !Src || Bytes <= 0) {
+ if (Bytes == 0) {
+ ODBG(ADT_Interface) << "Zero bytes, nothing to do";
+ return;
+ }
+ REPORT() << "Invalid arguments";
+ return;
+ }
+
+ auto DstDeviceOrErr = DM->getDevice();
+ if (!DstDeviceOrErr)
+ REPORT_FATAL() << toString(DstDeviceOrErr.takeError()).c_str();
+ auto SrcDeviceOrErr = DM->getDevice();
+ if (!SrcDeviceOrErr)
+ REPORT_FATAL() << toString(SrcDeviceOrErr.takeError()).c_str();
+ if (!SrcDeviceOrErr->isDataExchangable(*DstDeviceOrErr)) {
+ REPORT() << "D2D not allowed for current device type";
+ return;
+ }
+
+ AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);
+ int Rc =
+ SrcDeviceOrErr->dataExchange(Src, *DstDeviceOrErr, Dst, Bytes, AsyncInfo);
+ ODBG(ADT_Interface) << "Result " << Rc;
+}
+
+void accMapData(void *Hst, void *Dev, size_t Bytes) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << Hst << " <-> " << Dev << ", " << Bytes << " bytes";
+
+ if (!Hst || !Dev || Bytes <= 0) {
+ REPORT() << "Invalid arguments";
+ return;
+ }
+
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+
+ int Rc = DeviceOrErr->getMappingInfo().associatePtr(
+ const_cast<void *>(Hst), const_cast<void *>(Dev), Bytes);
+ ODBG(ADT_Interface) << "Result " << Rc;
+}
+
+void accUnmapData(void *Hst) {
+ FUNC_LOGGER();
+ ODBG(ADT_Interface) << Hst;
+
+ if (!Hst) {
+ REPORT() << "Invalid arguments";
+ return;
+ }
+
+ auto DeviceOrErr = DM->getDevice();
+ if (!DeviceOrErr)
+ REPORT_FATAL() << toString(DeviceOrErr.takeError()).c_str();
+ int Rc =
+ DeviceOrErr->getMappingInfo().disassociatePtr(const_cast<void *>(Hst));
+ ODBG(ADT_Interface) << "Result " << Rc;
+}
+
+} // namespace llvm::acc::target
diff --git a/offload/libacctarget/RuntimeInterface.cpp b/offload/libacctarget/RuntimeInterface.cpp
new file mode 100644
index 00000000000000..83c4b196155ed3
--- /dev/null
+++ b/offload/libacctarget/RuntimeInterface.cpp
@@ -0,0 +1,249 @@
+//===- AccEntryCommonImpl.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DeviceManager.h"
+#include "Interface.h"
+#include "Private.h"
+#include "QueueManager.h"
+#include "Shared/Debug.h"
+#include "include/openacc.h"
+
+using namespace llvm::acc::target;
+
+extern "C" {
+int acc_get_num_devices(acc_device_t DevType) {
+ return DM->getNumDevices(DevType);
+}
+int acc_get_num_devices_(acc_device_t *DevType) {
+ return acc_get_num_devices(*DevType);
+}
+
+int acc_get_device_num(acc_device_t DevType) {
+ return DM->getDeviceId(DevType);
+}
+int acc_get_device_num_(acc_device_t *DevType) {
+ return acc_get_device_num(*DevType);
+}
+
+void acc_set_device_num(int DevNum, acc_device_t DevType) {
+ __tgt_acc_set_device_num(nullptr, 0, DevType, DevNum);
+}
+void acc_set_device_num_(int *DevNum, acc_device_t *DevType) {
+ acc_set_device_num(*DevNum, *DevType);
+}
+
+void acc_set_device_type(acc_device_t DevType) {
+ __tgt_acc_set_device_type(nullptr, 0, DevType);
+}
+void acc_set_device_type_(acc_device_t *DevType) {
+ acc_set_device_type(*DevType);
+}
+
+void acc_set_device(acc_device_t DevType) {
+ __tgt_acc_set_device_type(nullptr, 0, DevType);
+}
+void acc_set_device_(acc_device_t *DevType) { acc_set_device(*DevType); }
+
+acc_device_t acc_get_device_type(void) { return DM->getDeviceType(); }
+acc_device_t acc_get_device_type_(void) { return acc_get_device_type(); }
+
+acc_device_t acc_get_device(void) { return DM->getDeviceType(); }
+acc_device_t acc_get_device_(void) { return acc_get_device(); }
+
+size_t acc_get_property(int DevNum, acc_device_t DevType,
+ acc_device_property_t Prop) {
+ return DM->getDeviceProperty(DevNum, DevType, Prop);
+}
+size_t acc_get_property_(int *DevNum, acc_device_t *DevType,
+ acc_device_property_t *Prop) {
+ return acc_get_property(*DevNum, *DevType, *Prop);
+}
+
+const char *acc_get_property_string(int DevNum, acc_device_t DevType,
+ acc_device_property_t Prop) {
+ return DM->getDevicePropertyString(DevNum, DevType, Prop);
+}
+const char *acc_get_property_string_(int *DevNum, acc_device_t *DevType,
+ acc_device_property_t *Prop) {
+ return acc_get_property_string(*DevNum, *DevType, *Prop);
+}
+
+void acc_async_wait(int WaitArg) {
+ accAsyncWait(nullptr, DM->getPMDeviceId(), WaitArg);
+}
+void acc_async_wait_(int WaitArg) { acc_async_wait(WaitArg); }
+
+void acc_wait_async(int WaitArg, int Async) {
+ (void)Async;
+ accAsyncWait(nullptr, DM->getPMDeviceId(), WaitArg);
+}
+void acc_wait_async_(int WaitArg, int Async) { acc_wait_async(WaitArg, Async); }
+
+void acc_wait(int WaitArg) {
+ accAsyncWait(nullptr, DM->getPMDeviceId(), WaitArg);
+}
+void acc_wait_(int WaitArg) { acc_wait(WaitArg); }
+
+void acc_wait_device(int WaitArg, int DevNum) {
+ accAsyncWait(nullptr, DevNum, WaitArg);
+}
+void acc_wait_device_(int WaitArg, int DevNum) {
+ acc_wait_device(WaitArg, DevNum);
+}
+
+void acc_wait_all_async() { accAsyncWaitAll(nullptr); }
+void acc_wait_all_async_() { acc_wait_all_async(); }
+
+void acc_async_wait_all() { accAsyncWaitAll(nullptr); }
+void acc_async_wait_all_() { acc_async_wait_all(); }
+
+void acc_wait_all() { accAsyncWaitAll(nullptr); }
+void acc_wait_all_() { acc_wait_all(); }
+
+void acc_wait_all_device(int DevNum) { accAsyncWaitAll(nullptr, DevNum); }
+void acc_wait_all_device_(int *DevNum) { acc_wait_all_device(*DevNum); }
+
+int acc_wait_any(int Count, int *WaitNum) {
+ REPORT_FATAL() << "acc_wait_any not yet implemented.";
+ return 0;
+}
+int acc_wait_any_(int *Count, int **WaitNum) {
+ return acc_wait_any(*Count, *WaitNum);
+}
+
+int acc_wait_any_device(int Count, int *WaitNum, int DevNum) {
+ REPORT_FATAL() << "acc_wait_any_device not yet implemented.";
+ return 0;
+}
+int acc_wait_any_device_(int *Count, int **WaitNum, int *DevNum) {
+ return acc_wait_any_device(*Count, *WaitNum, *DevNum);
+}
+
+void acc_set_default_async(int Async) {
+ __tgt_acc_set_default_async(nullptr, Async);
+}
+void acc_set_default_async_(int Async) { acc_set_default_async(Async); }
+
+int acc_get_default_async(void) { return icv::AccDefaultAsyncVar; }
+int acc_get_default_async_(void) { return acc_get_default_async(); }
+
+int acc_async_test(int TestArg) {
+ return !accAsyncTest(nullptr, DM->getPMDeviceId(), TestArg);
+};
+int acc_async_test_(int WaitArg) { return acc_async_test(WaitArg); }
+
+int acc_async_test_device(int DevNum, int WaitArg) {
+ return !accAsyncTest(nullptr, DevNum, WaitArg);
+}
+int acc_async_test_device_(int DevNum, int WaitArg) {
+ return acc_async_test_device(DevNum, WaitArg);
+}
+
+int acc_async_test_all(void) { return !accAsyncTestAll(nullptr); }
+int acc_async_test_all_(void) { return acc_async_test_all(); }
+
+int acc_async_test_all_device(int DevNum) {
+ return !accAsyncTestAll(nullptr, DevNum);
+}
+int acc_async_test_all_device_(int *DevNum) {
+ return acc_async_test_all_device(*DevNum);
+}
+
+void acc_init(acc_device_t DevType) { __tgt_acc_init(nullptr, 0, DevType, -1); }
+void acc_init_(acc_device_t *DevType) { acc_init(*DevType); }
+
+void acc_init_device(int DevNum, acc_device_t DevType) {
+ __tgt_acc_init(nullptr, 0, DevType, DevNum);
+}
+void acc_init_device_(int *DevNum, acc_device_t *DevType) {
+ acc_init_device(*DevNum, *DevType);
+}
+
+void acc_shutdown(acc_device_t DevType) {
+ __tgt_acc_shutdown(nullptr, 0, DevType, -1);
+}
+void acc_shutdown_(acc_device_t *DevType) { acc_shutdown(*DevType); }
+
+void acc_shutdown_device(int DevNum, acc_device_t DevType) {
+ __tgt_acc_shutdown(nullptr, 0, DevType, DevNum);
+}
+void acc_shutdown_device_(int *DevNum, acc_device_t *DevType) {
+ acc_shutdown_device(*DevNum, *DevType);
+}
+
+void acc_free(void *DataDev) { accFree(DataDev); }
+void acc_free_(void **DataDev) { acc_free(*DataDev); }
+
+void *acc_malloc(size_t Bytes) { return accAlloc(Bytes); }
+void *acc_malloc_(size_t *Bytes) { return acc_malloc(*Bytes); }
+
+void acc_map_data(void *DataArg, void *DataDev, size_t Bytes) {
+ accMapData(DataArg, DataDev, Bytes);
+}
+void acc_map_data_(void **DataArg, void **DataDev, size_t *Bytes) {
+ acc_map_data(*DataArg, *DataDev, *Bytes);
+}
+
+void acc_unmap_data(void *DataArg) { accUnmapData(DataArg); }
+void acc_unmap_data_(void **DataArg) { acc_unmap_data(*DataArg); }
+
+void *acc_deviceptr(void *DataArg) {
+ return __tgt_acc_get_deviceptr(nullptr, DataArg, 0, DataArg);
+}
+void *acc_deviceptr_(void **DataArg) { return acc_deviceptr(*DataArg); }
+
+void *acc_hostptr(void *DataDev) {
+ REPORT_FATAL() << "acc_hostptr not yet implemented";
+ return nullptr;
+}
+void *acc_hostptr_(void **DataDev) { return acc_hostptr(*DataDev); }
+
+void acc_memcpy_from_device(void *DataHostDest, void *DataDevSrc,
+ size_t Bytes) {
+ accMemcpyFromDevice(DataHostDest, DataDevSrc, Bytes);
+}
+void acc_memcpy_from_device_(void **DataHostDest, void **DataDevSrc,
+ size_t *Bytes) {
+ acc_memcpy_from_device(*DataHostDest, *DataDevSrc, *Bytes);
+}
+
+void acc_memcpy_to_device(void *DataDevDest, void *DataHostSrc, size_t Bytes) {
+ accMemcpyToDevice(DataDevDest, DataHostSrc, Bytes);
+}
+void acc_memcpy_to_device_(void **DataDevDest, void **DataHostSrc,
+ size_t *Bytes) {
+ acc_memcpy_to_device(*DataDevDest, *DataHostSrc, *Bytes);
+}
+
+void acc_memcpy_d2d(void *DataDevDest, void *DataHostSrc, size_t Bytes,
+ int DevNumDest, int DevNumSrc) {
+ accMemcpyD2D(DataDevDest, DataHostSrc, Bytes, DevNumDest, DevNumSrc);
+}
+void acc_memcpy_d2d_(void **DataDevDest, void **DataHostSrc, size_t *Bytes,
+ int *DevNumDest, int *DevNumSrc) {
+ acc_memcpy_d2d(*DataDevDest, *DataHostSrc, *Bytes, *DevNumDest, *DevNumSrc);
+}
+
+int acc_on_device(acc_device_t DevType) { return DevType == acc_device_host; }
+int acc_on_device_(acc_device_t *DevType) { return acc_on_device(*DevType); }
+
+void acc_present_dump_all() {
+ REPORT_WARN() << "acc_present_dump_all not yet implemented";
+}
+void acc_present_dump_all_() { acc_present_dump_all(); }
+
+void acc_attach_dump_all() {
+ REPORT_WARN() << "acc_attach_dump_all not yet implemented";
+}
+void acc_attach_dump_all_() { acc_attach_dump_all(); }
+
+void acc_attach_dump() {
+ REPORT_WARN() << "acc_attach_dump not yet implemented";
+}
+void acc_attach_dump_() { acc_attach_dump(); }
+}
diff --git a/offload/libacctarget/RuntimeState.cpp b/offload/libacctarget/RuntimeState.cpp
new file mode 100644
index 00000000000000..e7e756a0569e53
--- /dev/null
+++ b/offload/libacctarget/RuntimeState.cpp
@@ -0,0 +1,80 @@
+//===-------- RuntimeState.cpp ------------------------------------ C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Contains managing the OpenACC runtime state.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RuntimeState.h"
+#include "Debug.h"
+#include "DeviceManager.h"
+#include "Logger.h"
+#include "QueueManager.h"
+
+#include <cassert>
+#include <mutex>
+
+namespace llvm::acc::target {
+PluginManager *PM = nullptr;
+} // namespace llvm::acc::target
+
+using namespace llvm::acc::target;
+using namespace llvm::acc::target::debug;
+
+static std::mutex &getInitMutex() {
+ static std::mutex Mutex;
+ return Mutex;
+}
+static int32_t InitRefCount = 0;
+
+void initAccRuntime() {
+ FUNC_LOGGER();
+ std::scoped_lock<std::mutex> Lock(getInitMutex());
+ assert(InitRefCount >= 0 && "Ref count cannot be negative.");
+
+ if (InitRefCount++ == 0) {
+ ODBG(ADT_Init) << "Init OpenACC runtime";
+ assert(!PM && "PM is present");
+ PM = new PluginManager();
+ PM->init();
+ // TODO Blindly register all rtls for now. In reality we should only be
+ // initializing the requested types in case we come from __tgt_acc_init(),
+ // or only the ones we have device code for.
+ PM->initializeAllDevices();
+ PM->registerDelayedLibraries();
+
+ DM = new DeviceManagerTy();
+ DM->init();
+
+ QueueManager = new QueueManagerTy();
+ QueueManager->init();
+ }
+ DM->refreshDeviceMapping(/*UpdateDeviceType=*/true);
+}
+
+void deinitAccRuntime() {
+ FUNC_LOGGER();
+ std::scoped_lock<std::mutex> Lock(getInitMutex());
+ assert(PM && "OpenACC runtime not initialized");
+ assert(InitRefCount > 0 && "Ref count cannot be negative.");
+
+ if (--InitRefCount == 0) {
+ ODBG(ADT_Init) << "Deinit OpenACC runtime";
+ QueueManager->deinit();
+ delete QueueManager;
+ QueueManager = nullptr;
+
+ DM->deinit();
+ delete DM;
+ DM = nullptr;
+
+ PM->deinit();
+ delete PM;
+ PM = nullptr;
+ }
+}
diff --git a/offload/libacctarget/RuntimeState.h b/offload/libacctarget/RuntimeState.h
new file mode 100644
index 00000000000000..6f464595c80a2e
--- /dev/null
+++ b/offload/libacctarget/RuntimeState.h
@@ -0,0 +1,24 @@
+//===-------- RuntimeState.h -------------------------------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBACCTARGET_RUNTIMESTATE_H_
+#define _LIBACCTARGET_RUNTIMESTATE_H_
+
+#include "PluginManager.h"
+
+namespace llvm::acc::target {
+extern PluginManager *PM;
+} // namespace llvm::acc::target
+
+/// Initialize the OpenACC runtime.
+void initAccRuntime();
+
+/// Completely deinitialize the OpenACC runtime.
+void deinitAccRuntime();
+
+#endif // _LIBACCTARGET_RUNTIMESTATE_H_
diff --git a/offload/libacctarget/exports b/offload/libacctarget/exports
new file mode 100644
index 00000000000000..9b8dbfc2b35021
--- /dev/null
+++ b/offload/libacctarget/exports
@@ -0,0 +1,191 @@
+VERS1.0 {
+ global:
+ __tgt_acc_rtl_init;
+ __tgt_acc_rtl_deinit;
+ __tgt_acc_register_lib;
+ __tgt_acc_unregister_lib;
+ __acc_compiled;
+ __tgt_acc_register_rpc_callback;
+
+ acc_is_present;
+ _cfi_acc_is_present_a;
+
+ acc_create;
+ acc_pcreate;
+ acc_present_or_create;
+ acc_delete;
+ acc_delete_finalize;
+ acc_copyin;
+ acc_pcopyin;
+ acc_present_or_copyin;
+ acc_copyout;
+ acc_copyout_finalize;
+ acc_update_device;
+ acc_updatein;
+ acc_update_self;
+ acc_update_host;
+ acc_updateout;
+
+ acc_create_async;
+ acc_pcreate_async;
+ acc_present_or_create_async;
+ acc_delete_async;
+ acc_delete_finalize_async;
+ acc_copyin_async;
+ acc_pcopyin_async;
+ acc_present_or_copyin_async;
+ acc_copyout_async;
+ acc_copyout_finalize_async;
+ acc_update_device_async;
+ acc_updatein_async;
+ acc_update_self_async;
+ acc_update_host_async;
+ acc_updateout_async;
+
+ _cfi_acc_create_a;
+ _cfi_acc_pcreate_a;
+ _cfi_acc_present_or_create_a;
+ _cfi_acc_delete_a;
+ _cfi_acc_delete_finalize_a;
+ _cfi_acc_copyin_a;
+ _cfi_acc_pcopyin_a;
+ _cfi_acc_present_or_copyin_a;
+ _cfi_acc_copyout_a;
+ _cfi_acc_copyout_finalize_a;
+ _cfi_acc_update_device_a;
+ _cfi_acc_updatein_a;
+ _cfi_acc_update_self_a;
+ _cfi_acc_update_host_a;
+ _cfi_acc_updateout_a;
+
+ _cfi_acc_create_async_a;
+ _cfi_acc_pcreate_async_a;
+ _cfi_acc_present_or_create_async_a;
+ _cfi_acc_delete_async_a;
+ _cfi_acc_delete_finalize_async_a;
+ _cfi_acc_copyin_async_a;
+ _cfi_acc_pcopyin_async_a;
+ _cfi_acc_present_or_copyin_async_a;
+ _cfi_acc_copyout_async_a;
+ _cfi_acc_copyout_finalize_async_a;
+ _cfi_acc_update_device_async_a;
+ _cfi_acc_updatein_async_a;
+ _cfi_acc_update_self_async_a;
+ _cfi_acc_update_host_async_a;
+ _cfi_acc_updateout_async_a;
+
+ __tgt_acc_declare;
+ __tgt_acc_mirror_alloc;
+ __tgt_acc_mirror_dealloc;
+ __tgt_acc_data_update;
+ __tgt_acc_copy_present_d2h;
+ __tgt_acc_data_enter;
+ __tgt_acc_data_exit;
+ __tgt_acc_data_begin;
+ __tgt_acc_data_end;
+ __tgt_acc_kernel;
+ __tgt_acc_alloc;
+ __tgt_acc_free;
+ __tgt_acc_get_deviceptr;
+ __tgt_acc_set_default_async;
+ __tgt_acc_set_device_num;
+ __tgt_acc_set_device_type;
+ __tgt_acc_wait;
+ __tgt_acc_init;
+ __tgt_acc_shutdown;
+
+ acc_get_num_devices;
+ acc_get_num_devices_;
+ acc_get_device_num;
+ acc_get_device_num_;
+ acc_set_device_num;
+ acc_set_device_num_;
+ acc_set_device_type;
+ acc_set_device_type_;
+ acc_set_device;
+ acc_set_device_;
+ acc_get_device_type;
+ acc_get_device_type_;
+ acc_get_device;
+ acc_get_device_;
+ acc_get_property;
+ acc_get_property_;
+ acc_get_property_string;
+ acc_get_property_string_;
+
+ acc_async_wait;
+ acc_async_wait_;
+ acc_wait_async;
+ acc_wait_async_;
+ acc_wait;
+ acc_wait_;
+ acc_wait_device;
+ acc_wait_device_;
+ acc_wait_all_async;
+ acc_wait_all_async_;
+ acc_async_wait_all;
+ acc_async_wait_all_;
+ acc_wait_all;
+ acc_wait_all_;
+ acc_wait_all_device;
+ acc_wait_all_device_;
+ acc_wait_any;
+ acc_wait_any_;
+ acc_wait_any_device;
+ acc_wait_any_device_;
+
+ acc_async_test;
+ acc_async_test_;
+ acc_async_test_device;
+ acc_async_test_device_;
+ acc_async_test_all;
+ acc_async_test_all_;
+ acc_async_test_all_device;
+ acc_async_test_all_device_;
+
+ acc_init;
+ acc_init_;
+ acc_init_device;
+ acc_init_device_;
+ acc_shutdown;
+ acc_shutdown_;
+ acc_shutdown_device;
+ acc_shutdown_device_;
+
+ acc_set_default_async;
+ acc_set_default_async_;
+ acc_get_default_async;
+ acc_get_default_async_;
+
+ acc_malloc;
+ acc_malloc_;
+ acc_free;
+ acc_free_;
+ acc_map_data;
+ acc_map_data_;
+ acc_unmap_data;
+ acc_unmap_data_;
+ acc_deviceptr;
+ acc_deviceptr_;
+ acc_hostptr;
+ acc_hostptr_;
+
+ acc_memcpy_from_device;
+ acc_memcpy_from_device_;
+ acc_memcpy_to_device;
+ acc_memcpy_to_device_;
+ acc_memcpy_d2d;
+ acc_memcpy_d2d_;
+
+ acc_on_device;
+ acc_on_device_;
+ acc_present_dump_all;
+ acc_present_dump_all_;
+ acc_attach_dump_all;
+ acc_attach_dump_all_;
+ acc_attach_dump;
+ acc_attach_dump_;
+
+ local:
+ *;
+};
diff --git a/offload/libacctarget/include/openacc.h b/offload/libacctarget/include/openacc.h
new file mode 100644
index 00000000000000..8f77d220a8d8e4
--- /dev/null
+++ b/offload/libacctarget/include/openacc.h
@@ -0,0 +1,46 @@
+//===- openacc.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ACC_OFFLOAD_INCLUDE_OPENACC_H_
+#define LLVM_ACC_OFFLOAD_INCLUDE_OPENACC_H_
+
+#define acc_async_sync -1
+#define acc_async_default -3
+#define acc_async_noval -4
+
+typedef enum {
+ acc_device_none = 0,
+ acc_device_default = 1,
+ acc_device_host = 2,
+ acc_device_not_host = 3,
+ acc_device_current = 10,
+
+ acc_device_concrete_type_begin = 4,
+ acc_device_nvidia = 4,
+ acc_device_amd = 5,
+ acc_device_spirv = 6,
+ acc_device_concrete_type_end = 7,
+
+} acc_device_t;
+
+typedef enum {
+ acc_property_int_begin = 0,
+ acc_property_memory = 0,
+ acc_property_free_memory = 1,
+ acc_property_shared_memory_support = 2,
+ acc_property_int_end = 3,
+
+ acc_property_string_begin = 1000,
+ acc_property_name = 1000,
+ acc_property_vendor = 1001,
+ acc_property_driver = 1002,
+ acc_property_string_end = 1003,
+
+} acc_device_property_t;
+
+#endif // LLVM_ACC_OFFLOAD_INCLUDE_OPENACC_H_
diff --git a/offload/libompaccsupport/AsyncInfo.cpp b/offload/libompaccsupport/AsyncInfo.cpp
index 5d0dcfabb1e20b..663507a2b5109d 100644
--- a/offload/libompaccsupport/AsyncInfo.cpp
+++ b/offload/libompaccsupport/AsyncInfo.cpp
@@ -15,6 +15,7 @@
#include <cstdint>
using namespace llvm::omp::target::debug;
+using llvm::SmallVector;
// Temporary helper from liboffload until all usage of AsyncInfo is migrated
// to liboffload queues.
@@ -32,7 +33,7 @@ AsyncInfoTy::AsyncInfoTy(DeviceTy &Device, SyncTy SyncType)
}
AsyncInfoTy::~AsyncInfoTy() {
- synchronize();
+ finalize();
if (Queue) {
if (auto Res = olDestroyQueue(Queue))
REPORT() << "Failed to destroy queue " << Queue << ": " << Res->Details;
@@ -43,7 +44,7 @@ AsyncInfoTy::operator __tgt_async_info *() {
return llvm::offload::tmp::__ol_tgt_GetAsyncInfoFromQueue(Queue);
}
-int AsyncInfoTy::synchronize() {
+int AsyncInfoTy::finalize() {
int Result = OFFLOAD_SUCCESS;
if (!isQueueEmpty()) {
switch (SyncType) {
@@ -54,6 +55,10 @@ int AsyncInfoTy::synchronize() {
case SyncTy::NON_BLOCKING:
Result = Device.queryAsync(*this);
break;
+ case SyncTy::STATIC_NON_BLOCKING:
+ Result = Device.queryAsyncStatic(*this) == OFFLOAD_FAIL ? OFFLOAD_FAIL
+ : OFFLOAD_SUCCESS;
+ break;
}
}
@@ -67,6 +72,48 @@ int AsyncInfoTy::synchronize() {
return Result;
}
+static int32_t processPPFs(SmallVector<AsyncInfoTy::PostProcFuncTy> &PPFs) {
+ for (size_t I = 0; I < PPFs.size(); ++I)
+ if (int Res = PPFs[I](); Res != OFFLOAD_SUCCESS)
+ return Res;
+ return OFFLOAD_SUCCESS;
+}
+
+int AsyncInfoTy::synchronize() {
+ assert(SyncType == SyncTy::STATIC_NON_BLOCKING);
+
+ // We still have not created a queue, or this specific device does not
+ // generate queues.
+ if (!Queue)
+ return OFFLOAD_SUCCESS;
+
+ // STATIC_NON_BLOCKING retains its queue thus more careful handling of the
+ // post processing functions is required.
+
+ // Collect the enqueued PPFs until this point.
+ SmallVector<PostProcFuncTy> LocalPPFs;
+ {
+ std::lock_guard<std::mutex> PPFGuard{PostProcessingFunctionsMutex};
+ std::swap(LocalPPFs, PostProcessingFunctions);
+ }
+ int Result = Device.synchronizeStatic(*this);
+ // Run any pending post-processing function collected _before_ we
+ // synchronize. This is important as between before the synchronization and
+ // after we could have enqueued more post processing operations, which we
+ // must not run yet.
+ if (Result == OFFLOAD_SUCCESS)
+ Result = processPPFs(LocalPPFs);
+
+ return Result;
+}
+
+int AsyncInfoTy::query() {
+ // If we don't have a queue, there are no pending actions.
+ if (!Queue)
+ return 0;
+ return Device.queryAsyncStatic(*this);
+}
+
void *&AsyncInfoTy::getVoidPtrLocation() {
BufferLocations.push_back(nullptr);
return BufferLocations.back();
@@ -75,19 +122,20 @@ void *&AsyncInfoTy::getVoidPtrLocation() {
bool AsyncInfoTy::isDone() const { return isQueueEmpty(); }
int32_t AsyncInfoTy::runPostProcessing() {
- size_t Size = PostProcessingFunctions.size();
- for (size_t I = 0; I < Size; ++I) {
- const int Result = PostProcessingFunctions[I]();
+ // Post-processing procedures might add new procedures themselves, so
+ // repeatedly process them until we are done.
+ while (true) {
+ SmallVector<PostProcFuncTy> LocalPPFs;
+ {
+ std::lock_guard<std::mutex> PPFGuard{PostProcessingFunctionsMutex};
+ std::swap(LocalPPFs, PostProcessingFunctions);
+ }
+ if (LocalPPFs.size() == 0)
+ return OFFLOAD_SUCCESS;
+ int32_t Result = processPPFs(LocalPPFs);
if (Result != OFFLOAD_SUCCESS)
return Result;
}
-
- // Clear the vector up until the last known function, since post-processing
- // procedures might add new procedures themselves.
- const auto *PrevBegin = PostProcessingFunctions.begin();
- PostProcessingFunctions.erase(PrevBegin, PrevBegin + Size);
-
- return OFFLOAD_SUCCESS;
}
bool AsyncInfoTy::isQueueEmpty() const {
diff --git a/offload/libompaccsupport/CMakeLists.txt b/offload/libompaccsupport/CMakeLists.txt
new file mode 100644
index 00000000000000..ff6c4deb598c39
--- /dev/null
+++ b/offload/libompaccsupport/CMakeLists.txt
@@ -0,0 +1,9 @@
+set(LIBOMPACCSUPPORT_SOURCES
+ ${CMAKE_CURRENT_SOURCE_DIR}/AsyncInfo.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/PluginManager.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/DeviceImage.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/Mapping.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/OmpAccError.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/Program.cpp
+ PARENT_SCOPE)
diff --git a/offload/libompaccsupport/Mapping.cpp b/offload/libompaccsupport/Mapping.cpp
index 37ec36670d6fd2..f36a5970dbe793 100644
--- a/offload/libompaccsupport/Mapping.cpp
+++ b/offload/libompaccsupport/Mapping.cpp
@@ -14,6 +14,10 @@
#include "Shared/Debug.h"
#include "Shared/Requirements.h"
#include "device.h"
+#include "omptarget.h"
+#include <cstdint>
+#include <optional>
+#include <variant>
using namespace llvm::omp::target::debug;
@@ -206,12 +210,22 @@ LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
TargetPointerResultTy MappingInfoTy::getTargetPointer(
HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
- int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
- bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
- bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+ int64_t TgtPadding, std::variant<int64_t, const NonContigDescTy *> MemInfo,
+ map_var_info_t HstPtrName, bool HasFlagTo, bool HasFlagAlways,
+ bool IsImplicit, bool UpdateRefCount, bool HasCloseModifier,
+ bool HasPresentModifier, bool HasHoldModifier, bool IsNoCreate,
AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap,
StateInfoTy *StateInfo, const ident_t *Loc) {
+ int64_t Size;
+ const NonContigDescTy *CopyInfo = nullptr;
+ if (std::holds_alternative<int64_t>(MemInfo)) {
+ Size = std::get<int64_t>(MemInfo);
+ } else {
+ CopyInfo = std::get<const NonContigDescTy *>(MemInfo);
+ Size = CopyInfo->getAllocSize();
+ }
+
LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
LR.TPR.Flags.IsPresent = true;
@@ -250,7 +264,8 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
(HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
LR.TPR.TargetPointer = (void *)Ptr;
- } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
+ } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit &&
+ !IsNoCreate) {
// Explicit extension of mapped data - not allowed.
MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
" (%" PRId64
@@ -290,7 +305,7 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
MESSAGE("device mapping required by 'present' map type modifier does not "
"exist for host address " DPxMOD " (%" PRId64 " bytes)",
DPxPTR(HstPtrBegin), Size);
- } else if (Size) {
+ } else if (Size && !IsNoCreate) {
// If it is not contained and Size > 0, we should create a new entry for it.
LR.TPR.Flags.IsNewEntry = true;
uintptr_t TgtAllocBegin =
@@ -329,6 +344,12 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
if (ReleaseHDTTMap)
HDTTMap.destroy();
+ if (!LR.TPR.isPresent() && IsNoCreate) {
+ ODBG() << "Mapping for " << HstPtrBegin << " with size " << Size
+ << " does not exist and no_create is specified: returning.";
+ return std::move(LR.TPR);
+ }
+
// Lambda to check if this pointer was newly allocated on the current region.
// This is needed to handle cases when the TO entry is encountered after an
// alloc entry for the same pointer. In such cases, the ref-count is already
@@ -384,9 +405,15 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
ODBG(ODT_Mapping) << "Moving " << Size << " bytes (hst:" << HstPtrBegin
<< ") -> (tgt:" << LR.TPR.TargetPointer << ")";
- int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
- AsyncInfo, LR.TPR.getEntry(),
- /*HDTTMapPtr=*/nullptr, Loc);
+ int Ret;
+ if (CopyInfo) {
+ Ret = Device.submitNonContigData(LR.TPR.TargetPointer, HstPtrBegin,
+ *CopyInfo, AsyncInfo, LR.TPR.getEntry());
+ } else {
+ Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
+ AsyncInfo, LR.TPR.getEntry(),
+ /*HDTTMapPtr=*/nullptr, Loc);
+ }
if (Ret != OFFLOAD_SUCCESS) {
REPORT() << "Copying data to device failed.";
// We will also return nullptr if the data movement fails because that
@@ -556,6 +583,27 @@ int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
return Ret;
}
+static void printNonContigCopyInfoImpl(int DeviceId, bool H2D,
+ void *SrcPtrBegin, void *DstPtrBegin,
+ const NonContigDescTy &CopyInfo,
+ HostDataToTargetTy *HT) {
+
+ INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
+ "Copying non-contiguous data from %s to %s, %sPtr=" DPxMOD
+ ", %sPtr=" DPxMOD ", Name=%s\n",
+ H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
+ DPxPTR(H2D ? SrcPtrBegin : DstPtrBegin), H2D ? "Tgt" : "Hst",
+ DPxPTR(H2D ? DstPtrBegin : SrcPtrBegin),
+ (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
+ : "unknown");
+ ODBG(ODT_Mapping) << "Non-contiguous data descriptor:\n";
+ for (unsigned I = 0; I < CopyInfo.getRank(); I++)
+ ODBG(ODT_Mapping) << " Dim " << I << " : Offset "
+ << CopyInfo.Dims[I].Offset << " Count "
+ << CopyInfo.Dims[I].Count << " Stride "
+ << CopyInfo.Dims[I].Stride << "\n";
+}
+
static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
void *DstPtrBegin, int64_t Size,
HostDataToTargetTy *HT, const ident_t *Loc,
@@ -578,6 +626,16 @@ static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
LocStr.c_str());
}
+void MappingInfoTy::printNonContigCopyInfo(
+ void *TgtPtrBegin, void *HstPtrBegin, const NonContigDescTy &CopyInfo,
+ bool H2D, HostDataToTargetTy *Entry,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+ auto HDTTMap =
+ HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
+ printNonContigCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin,
+ CopyInfo, Entry);
+}
+
void MappingInfoTy::printCopyInfo(void *TgtPtrBegin, void *HstPtrBegin,
int64_t Size, bool H2D,
HostDataToTargetTy *Entry,
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index 7fc463c5943981..6157ba5d6cfc2f 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -385,7 +385,8 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
TblMapMtx.lock();
for (llvm::offloading::EntryTy *Cur = Desc->HostEntriesBegin;
Cur < Desc->HostEntriesEnd; ++Cur) {
- if (Cur->Kind == object::OffloadKind::OFK_OpenMP)
+ if (Cur->Kind == object::OffloadKind::OFK_OpenMP ||
+ Cur->Kind == object::OffloadKind::OFK_OpenACC)
HostPtrToTableMap.erase(Cur->Address);
}
@@ -454,7 +455,8 @@ int PluginManager::loadImagesOntoDevice(DeviceTy &Device) {
TransTable->TargetsEntries[DeviceId];
for (llvm::offloading::EntryTy &Entry :
llvm::make_range(Img->EntriesBegin, Img->EntriesEnd)) {
- if (Entry.Kind != object::OffloadKind::OFK_OpenMP)
+ if (Entry.Kind != object::OffloadKind::OFK_OpenMP &&
+ Entry.Kind != object::OffloadKind::OFK_OpenACC)
continue;
ProgramTy &Program = *LoadedOrErr;
@@ -586,7 +588,8 @@ int PluginManager::loadImagesOntoDevice(DeviceTy &Device) {
CurrDeviceEntry != EntryDeviceEnd;
CurrDeviceEntry++, CurrHostEntry++) {
if (CurrDeviceEntry->Size == 0 ||
- CurrDeviceEntry->Kind != object::OffloadKind::OFK_OpenMP)
+ (CurrDeviceEntry->Kind != object::OffloadKind::OFK_OpenMP &&
+ CurrDeviceEntry->Kind != object::OffloadKind::OFK_OpenACC))
continue;
assert(CurrDeviceEntry->Size == CurrHostEntry->Size &&
@@ -609,7 +612,7 @@ int PluginManager::loadImagesOntoDevice(DeviceTy &Device) {
void *DevPtr;
Device.retrieveData(&DevPtr, CurrDeviceEntryAddr, sizeof(void *),
AsyncInfo, /*Entry=*/nullptr, &HDTTMap);
- if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS)
+ if (AsyncInfo.finalize() != OFFLOAD_SUCCESS)
return OFFLOAD_FAIL;
CurrDeviceEntryAddr = DevPtr;
}
@@ -686,3 +689,53 @@ int PluginManager::getNumActivePlugins() const {
}
return count;
}
+
+TableMap *PluginManager::getTableMap(void *HostPtr) {
+ std::lock_guard<std::mutex> TblMapLock(TblMapMtx);
+ HostPtrToTableMapTy::iterator TableMapIt = HostPtrToTableMap.find(HostPtr);
+
+ if (TableMapIt != HostPtrToTableMap.end())
+ return &TableMapIt->second;
+
+ // We don't have a map. So search all the registered libraries.
+ TableMap *TM = nullptr;
+ std::lock_guard<std::mutex> TrlTblLock(TrlTblMtx);
+ for (HostEntriesBeginToTransTableTy::iterator Itr =
+ HostEntriesBeginToTransTable.begin();
+ Itr != HostEntriesBeginToTransTable.end(); ++Itr) {
+ // get the translation table (which contains all the good info).
+ TranslationTable *TransTable = &Itr->second;
+ // iterate over all the host table entries to see if we can locate the
+ // host_ptr.
+ llvm::offloading::EntryTy *Cur = TransTable->HostTable.EntriesBegin;
+ for (uint32_t I = 0; Cur < TransTable->HostTable.EntriesEnd; ++Cur, ++I) {
+ if (Cur->Address != HostPtr)
+ continue;
+ // we got a match, now fill the HostPtrToTableMap so that we
+ // may avoid this search next time.
+ TM = &(HostPtrToTableMap)[HostPtr];
+ TM->Table = TransTable;
+ TM->Index = I;
+ return TM;
+ }
+ }
+
+ return nullptr;
+}
+
+void llvm::omp::target::helpers::registerRPCCallback(
+ unsigned (*Callback)(void *, unsigned)) {
+ // Without an initialized liboffload there are no platforms to register with.
+ if (auto Err = iteratePlatforms(
+ [](ol_platform_handle_t Platform, void *Data) {
+ bool Active = false;
+ if (olGetPlatformInfo(Platform, OL_PLATFORM_INFO_ACTIVE,
+ sizeof(Active), &Active) == OL_SUCCESS &&
+ Active)
+ olPlatformRegisterRPCCallback(
+ Platform, reinterpret_cast<ol_platform_rpc_cb_t>(Data));
+ return true;
+ },
+ reinterpret_cast<void *>(Callback)))
+ consumeError(std::move(Err));
+}
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 27b790fad10355..d44e50081823e1 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -283,7 +283,8 @@ setupIndirectCallTable(DeviceTy &Device, __tgt_device_image *Image,
Image->EntriesEnd);
llvm::SmallVector<std::pair<void *, void *>> IndirectCallTable;
for (const auto &Entry : Entries) {
- if (Entry.Kind != llvm::object::OffloadKind::OFK_OpenMP ||
+ if ((Entry.Kind != llvm::object::OffloadKind::OFK_OpenMP &&
+ Entry.Kind != llvm::object::OffloadKind::OFK_OpenACC) ||
Entry.Size == 0 ||
(!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT) &&
!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE)))
@@ -548,6 +549,64 @@ int32_t DeviceTy::submitDataImpl(void *TgtPtrBegin, void *HstPtrBegin,
return OFFLOAD_SUCCESS;
}
+/// Copy a non-contiguous region chunk by chunk, walking the dimensions of
+/// \p CopyInfo. \p CopyChunk copies one contiguous chunk given an offset into
+/// the region.
+template <typename CopyFnTy>
+static int32_t copyNonContiguous(const NonContigDescTy &CopyInfo,
+ unsigned CurrentDim, uint64_t Offset,
+ CopyFnTy &CopyChunk) {
+ ODBG(ODT_Mapping) << "Non Contig Copy of Dim " << CurrentDim;
+ if (CurrentDim == CopyInfo.getRank())
+ return CopyChunk(Offset, CopyInfo.getLastDimCopySize());
+
+ for (unsigned int I = 0; I < CopyInfo.Dims[CurrentDim].Count; ++I) {
+ uint64_t CurOffset =
+ CopyInfo.Dims[CurrentDim].Offset + I * CopyInfo.Dims[CurrentDim].Stride;
+ // we only need to transfer the first element for the last dimension
+ // since we've already got a contiguous piece.
+ if (CurrentDim != CopyInfo.getRank() - 1 || I == 0) {
+ int32_t Ret = copyNonContiguous(CopyInfo, CurrentDim + 1,
+ Offset + CurOffset, CopyChunk);
+ if (Ret != OFFLOAD_SUCCESS)
+ return Ret;
+ }
+ }
+
+ return OFFLOAD_SUCCESS;
+}
+
+int32_t
+DeviceTy::submitNonContigData(void *TgtPtrBegin, void *HstPtrBegin,
+ const NonContigDescTy &CopyInfo,
+ AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+ ol_queue_handle_t Queue = AsyncInfo.getQueue();
+ if (!Queue)
+ return OFFLOAD_FAIL;
+
+ if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
+ MappingInfo.printNonContigCopyInfo(TgtPtrBegin, HstPtrBegin, CopyInfo,
+ /*H2D=*/true, Entry, HDTTMapPtr);
+
+ NonContigDescTy MergedCopyInfo = CopyInfo;
+ MergedCopyInfo.mergeContiguousDims();
+ auto CopyChunk = [&](uint64_t Offset, uint64_t Size) {
+ if (auto Res = olMemcpy(Queue, static_cast<char *>(TgtPtrBegin) + Offset,
+ DeviceHandle,
+ static_cast<const char *>(HstPtrBegin) + Offset,
+ /*SrcDevice=*/nullptr, Size)) {
+ REPORT() << "Failure to copy non-contiguous data from host to device. "
+ << "Pointers: host = " << HstPtrBegin
+ << ", device = " << TgtPtrBegin << ": " << Res->Details;
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+ };
+ return copyNonContiguous(MergedCopyInfo, /*CurrentDim=*/0, /*Offset=*/0,
+ CopyChunk);
+}
+
// Retrieve data from device
int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
int64_t Size, AsyncInfoTy &AsyncInfo,
@@ -597,6 +656,36 @@ int32_t DeviceTy::retrieveDataImpl(void *HstPtrBegin, void *TgtPtrBegin,
return OFFLOAD_SUCCESS;
}
+int32_t DeviceTy::retrieveNonContigData(
+ void *HstPtrBegin, void *TgtPtrBegin, const NonContigDescTy &CopyInfo,
+ AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry,
+ MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+ ol_queue_handle_t Queue = AsyncInfo.getQueue();
+ if (!Queue)
+ return OFFLOAD_FAIL;
+
+ if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
+ MappingInfo.printNonContigCopyInfo(TgtPtrBegin, HstPtrBegin, CopyInfo,
+ /*H2D=*/false, Entry, HDTTMapPtr);
+
+ NonContigDescTy MergedCopyInfo = CopyInfo;
+ MergedCopyInfo.mergeContiguousDims();
+ auto CopyChunk = [&](uint64_t Offset, uint64_t Size) {
+ if (auto Res = olMemcpy(Queue, static_cast<char *>(HstPtrBegin) + Offset,
+ /*DstDevice=*/nullptr,
+ static_cast<const char *>(TgtPtrBegin) + Offset,
+ DeviceHandle, Size)) {
+ REPORT() << "Failure to copy non-contiguous data from device to host. "
+ << "Pointers: host = " << HstPtrBegin
+ << ", device = " << TgtPtrBegin << ": " << Res->Details;
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+ };
+ return copyNonContiguous(MergedCopyInfo, /*CurrentDim=*/0, /*Offset=*/0,
+ CopyChunk);
+}
+
// Copy data from current device to destination device directly
int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
int64_t Size, AsyncInfoTy &AsyncInfo) {
@@ -1379,6 +1468,19 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
return OFFLOAD_SUCCESS;
}
+int32_t DeviceTy::enqueueHostCall(void (*Callback)(void *), void *UserData,
+ AsyncInfoTy &AsyncInfo) {
+ ol_queue_handle_t Queue = AsyncInfo.getQueue();
+ if (!Queue)
+ return OFFLOAD_FAIL;
+ if (auto Res = olLaunchHostFunction(Queue, Callback, UserData)) {
+ REPORT() << "Failure to enqueue host call in device " << DeviceID << ": "
+ << Res->Details;
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+}
+
int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) {
ol_queue_handle_t Queue = AsyncInfo.getQueue();
if (!Queue)
@@ -1391,6 +1493,12 @@ int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) {
return OFFLOAD_SUCCESS;
}
+int32_t DeviceTy::synchronizeStatic(AsyncInfoTy &AsyncInfo) {
+ // The queue is owned by the AsyncInfoTy and is never released by
+ // synchronization, so this is the same as the regular synchronization.
+ return synchronize(AsyncInfo);
+}
+
int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) {
ol_queue_handle_t Queue = AsyncInfo.getQueue();
if (!Queue)
@@ -1402,6 +1510,18 @@ int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) {
return OFFLOAD_SUCCESS;
}
+int32_t DeviceTy::queryAsyncStatic(AsyncInfoTy &AsyncInfo) {
+ ol_queue_handle_t Queue = AsyncInfo.getQueue();
+ if (!Queue)
+ return 0;
+ bool IsComplete = false;
+ if (auto Res = olQueryQueue(Queue, &IsComplete)) {
+ REPORT() << "Failure to query stream " << Queue << ": " << Res->Details;
+ return OFFLOAD_FAIL;
+ }
+ return IsComplete ? 0 : 1;
+}
+
int32_t DeviceTy::recordEvent(void **Event, AsyncInfoTy &AsyncInfo) {
ol_event_handle_t NewEvent;
if (auto Res =
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index a05db18a341d7a..ee275456676bea 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -14,13 +14,7 @@ add_library(omptarget SHARED
OpenMP/InteropAPI.cpp
OpenMP/OMPT/Callback.cpp
- ../libompaccsupport/AsyncInfo.cpp
- ../libompaccsupport/device.cpp
- ../libompaccsupport/PluginManager.cpp
- ../libompaccsupport/DeviceImage.cpp
- ../libompaccsupport/Mapping.cpp
- ../libompaccsupport/OmpAccError.cpp
- ../libompaccsupport/Program.cpp
+ ${LIBOMPACCSUPPORT_SOURCES}
KernelLanguage/API.cpp
)
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 028d94e80fe393..ea8e962e6578ea 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -30,6 +30,7 @@
#include <cstdint>
#include <cstdio>
#include <cstdlib>
+#include <dlfcn.h>
#include <memory>
#include <vector>
@@ -91,7 +92,7 @@ EXTERN void __tgt_rtl_deinit() { deinitRuntime(); }
/// adds a target shared library to the target execution image
EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) {
initRuntime();
- if (PM->delayRegisterLib(Desc))
+ if (PM->delayRegisterLib(__tgt_register_lib, Desc))
return;
PM->registerLib(Desc);
@@ -189,7 +190,7 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
if (Rc == OFFLOAD_SUCCESS) {
printSyncInfo(Loc, DeviceId, AsyncInfo);
- Rc = AsyncInfo.synchronize();
+ Rc = AsyncInfo.finalize();
}
}
@@ -444,7 +445,7 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
TIMESCOPE_WITH_DETAILS_AND_IDENT("Runtime: synchronize", "", Loc);
if (Rc == OFFLOAD_SUCCESS) {
printSyncInfo(Loc, DeviceId, AsyncInfo);
- Rc = AsyncInfo.synchronize();
+ Rc = AsyncInfo.finalize();
}
handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc);
@@ -557,7 +558,7 @@ EXTERN int __tgt_target_kernel_replay(
LoopTripCount, AsyncInfo, ReplayOutcome);
if (Rc == OFFLOAD_SUCCESS)
- Rc = AsyncInfo.synchronize();
+ Rc = AsyncInfo.finalize();
if (Rc != OFFLOAD_SUCCESS) {
ODBG(ODT_Interface) << "Kernel replay failed in device " << DeviceId;
@@ -629,7 +630,7 @@ EXTERN void __tgt_target_nowait_query(void **AsyncHandle) {
if (QueryCounter.isAboveThreshold())
AsyncInfo->SyncType = AsyncInfoTy::SyncTy::BLOCKING;
- if (AsyncInfo->synchronize())
+ if (AsyncInfo->finalize())
FATAL_MESSAGE0(1, "Error while querying the async queue for completion.\n");
// If there are device operations still pending, return immediately without
// deallocating the handle and increase the current thread query count.
@@ -649,21 +650,14 @@ EXTERN void __tgt_target_nowait_query(void **AsyncHandle) {
EXTERN void __tgt_register_rpc_callback(unsigned (*Callback)(void *,
unsigned)) {
- if (!PM || !PM->isLibOffloadInitialized())
- return;
-
- if (auto Err = iteratePlatforms(
- [](ol_platform_handle_t Platform, void *Data) {
- bool Active = false;
- if (olGetPlatformInfo(Platform, OL_PLATFORM_INFO_ACTIVE,
- sizeof(Active), &Active) == OL_SUCCESS &&
- Active)
- olPlatformRegisterRPCCallback(
- Platform, reinterpret_cast<ol_platform_rpc_cb_t>(Data));
- return true;
- },
- reinterpret_cast<void *>(Callback)))
- REPORT() << "Failed to iterate platforms: " << toString(std::move(Err));
+ registerRPCCallback(Callback);
+
+ // The OpenACC runtime has its own liboffload state, and this is the entry
+ // point the Fortran runtime finds when both runtimes are linked.
+ using AccRegisterFnTy = void (*)(unsigned (*)(void *, unsigned));
+ if (auto AccRegister = reinterpret_cast<AccRegisterFnTy>(
+ dlsym(RTLD_DEFAULT, "__tgt_acc_register_rpc_callback")))
+ AccRegister(Callback);
}
EXTERN void *__tgt_get_mapped_ptr(int64_t DeviceId, const void *HostPtr) {
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index b03ec4afa0891e..43c58066bca907 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -577,10 +577,12 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
// PTR_AND_OBJ entry is handled below, and so the allocation might fail
// when HasPresentModifier.
PointerTpr = Device.getMappingInfo().getTargetPointer(
- HDTTMap, HstPtrBase, HstPtrBase, /*TgtPadding=*/0, sizeof(void *),
+ HDTTMap, HstPtrBase, HstPtrBase, /*TgtPadding=*/0,
+ static_cast<int64_t>(sizeof(void *)),
/*HstPtrName=*/nullptr,
/*HasFlagTo=*/false, /*HasFlagAlways=*/false, IsImplicit, UpdateRef,
- HasCloseModifier, HasPresentModifier, HasHoldModifier, AsyncInfo,
+ HasCloseModifier, HasPresentModifier, HasHoldModifier,
+ /*IsNoCreate=*/false, AsyncInfo,
/*OwnedTPR=*/nullptr, /*ReleaseHDTTMap=*/false);
PointerTgtPtrBegin = PointerTpr.TargetPointer;
IsHostPtr = PointerTpr.Flags.IsHostPointer;
@@ -616,7 +618,8 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
auto TPR = Device.getMappingInfo().getTargetPointer(
HDTTMap, HstPtrBegin, HstPtrBase, TgtPadding, DataSize, HstPtrName,
HasFlagTo, HasFlagAlways, IsImplicit, UpdateRef, HasCloseModifier,
- HasPresentModifier, HasHoldModifier, AsyncInfo, PointerTpr.getEntry(),
+ HasPresentModifier, HasHoldModifier, /*IsNoCreate=*/false, AsyncInfo,
+ PointerTpr.getEntry(),
/*ReleaseHDTTMap=*/true, StateInfo, Loc);
void *TgtPtrBegin = TPR.TargetPointer;
IsHostPtr = TPR.Flags.IsHostPointer;
@@ -1580,39 +1583,7 @@ static bool isLambdaMapping(int64_t Mapping) {
namespace {
/// Find the table information in the map or look it up in the translation
/// tables.
-TableMap *getTableMap(void *HostPtr) {
- std::lock_guard<std::mutex> TblMapLock(PM->TblMapMtx);
- HostPtrToTableMapTy::iterator TableMapIt =
- PM->HostPtrToTableMap.find(HostPtr);
-
- if (TableMapIt != PM->HostPtrToTableMap.end())
- return &TableMapIt->second;
-
- // We don't have a map. So search all the registered libraries.
- TableMap *TM = nullptr;
- std::lock_guard<std::mutex> TrlTblLock(PM->TrlTblMtx);
- for (HostEntriesBeginToTransTableTy::iterator Itr =
- PM->HostEntriesBeginToTransTable.begin();
- Itr != PM->HostEntriesBeginToTransTable.end(); ++Itr) {
- // get the translation table (which contains all the good info).
- TranslationTable *TransTable = &Itr->second;
- // iterate over all the host table entries to see if we can locate the
- // host_ptr.
- llvm::offloading::EntryTy *Cur = TransTable->HostTable.EntriesBegin;
- for (uint32_t I = 0; Cur < TransTable->HostTable.EntriesEnd; ++Cur, ++I) {
- if (Cur->Address != HostPtr)
- continue;
- // we got a match, now fill the HostPtrToTableMap so that we
- // may avoid this search next time.
- TM = &(PM->HostPtrToTableMap)[HostPtr];
- TM->Table = TransTable;
- TM->Index = I;
- return TM;
- }
- }
-
- return nullptr;
-}
+TableMap *getTableMap(void *HostPtr) { return PM->getTableMap(HostPtr); }
/// A class manages private arguments in a target region.
class PrivateArgumentManagerTy {
diff --git a/offload/test/acc/init.c b/offload/test/acc/init.c
new file mode 100644
index 00000000000000..3ecc7975d062fb
--- /dev/null
+++ b/offload/test/acc/init.c
@@ -0,0 +1,22 @@
+// RUN: %libacctarget-compile-generic && env LIBOMPTARGET_DEBUG=1 %libacctarget-run-generic 2>&1 | %fcheck-generic
+// REQUIRES: libacctarget, libomptarget-debug
+
+#include <stdio.h>
+
+extern void __tgt_acc_rtl_init();
+extern void __tgt_acc_rtl_deinit();
+
+int main(void) {
+ // CHECK: Init OpenACC runtime
+ // CHECK-NOT: Init OpenACC runtime
+ // CHECK: Deinit OpenACC runtime
+ // CHECK-NOT: Deinit OpenACC runtime
+ __tgt_acc_rtl_init();
+ __tgt_acc_rtl_init();
+ __tgt_acc_rtl_deinit();
+ __tgt_acc_rtl_deinit();
+
+ // CHECK: PASS
+ printf("PASS\n");
+ return 0;
+}
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index f042b60d2f1a0c..1629d3024fb8ee 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -149,6 +149,9 @@ if config.libomptarget_debug:
if config.has_libomptarget_ompt:
config.available_features.add('ompt')
+if config.has_libacctarget:
+ config.available_features.add('libacctarget')
+
config.available_features.add(config.libomptarget_current_target)
amdgpu_target = 'amdgpu-amd-amdhsa'
@@ -535,6 +538,13 @@ for libomptarget_target in config.libomptarget_all_targets:
config.substitutions.append(("%flang-" + libomptarget_target, \
"echo ignored-command"))
+config.substitutions.append(("%libacctarget-compile-run-and-check-generic",
+ "%libacctarget-compile-generic && %libacctarget-run-generic | " +
+ config.libomptarget_filecheck + " %s"))
+config.substitutions.append(("%libacctarget-run-generic", "%t"))
+config.substitutions.append(("%libacctarget-compile-generic",
+ "%clang %flags %flags_clang %s -lacctarget -o %t"))
+
config.substitutions.append(("%clangxx", config.test_cxx_compiler))
config.substitutions.append(("%clang", config.test_c_compiler))
diff --git a/offload/test/lit.site.cfg.in b/offload/test/lit.site.cfg.in
index 011de60aa2c7f8..52a1147386e754 100644
--- a/offload/test/lit.site.cfg.in
+++ b/offload/test/lit.site.cfg.in
@@ -26,6 +26,7 @@ config.libomptarget_not = "@OFFLOAD_NOT_EXECUTABLE@"
config.offload_device_info = "@OFFLOAD_DEVICE_INFO_EXECUTABLE@"
config.libomptarget_debug = @LIBOMPTARGET_DEBUG@
config.has_libomptarget_ompt = @LIBOMPTARGET_OMPT_SUPPORT@
+config.has_libacctarget = @BUILD_LIBACCTARGET@
config.offload_tblgen = "@OFFLOAD_TBLGEN_EXECUTABLE@"
config.omp_kernel_replay = "@OMP_KERNEL_REPLAY@"
# Let the main config do the real work.
diff --git a/openacc/CMakeLists.txt b/openacc/CMakeLists.txt
new file mode 100644
index 00000000000000..83eb9887ec017b
--- /dev/null
+++ b/openacc/CMakeLists.txt
@@ -0,0 +1,40 @@
+cmake_minimum_required(VERSION 3.20.0)
+set(LLVM_SUBPROJECT_TITLE "OpenACC")
+
+set(LLVM_COMMON_CMAKE_UTILS ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
+include(${LLVM_COMMON_CMAKE_UTILS}/Modules/CMakePolicy.cmake
+ NO_POLICY_SCOPE)
+
+list(INSERT CMAKE_MODULE_PATH 0
+ "${LLVM_COMMON_CMAKE_UTILS}/Modules"
+ )
+
+if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
+ message(FATAL_ERROR "Standalone build is not supported. Use:\n"
+ " cmake <llvm-project>/runtimes -DLLVM_ENABLE_RUNTIMES=openacc")
+endif()
+
+include(GNUInstallDirs)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED NO)
+set(CMAKE_CXX_EXTENSIONS NO)
+
+if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
+ set(OPENACC_TARGET_SUBDIR "${LLVM_DEFAULT_TARGET_TRIPLE}")
+ if(OPENACC_LIBDIR_SUBDIR)
+ string(APPEND OPENACC_TARGET_SUBDIR "/${OPENACC_LIBDIR_SUBDIR}")
+ endif()
+ cmake_path(NORMAL_PATH OPENACC_TARGET_SUBDIR)
+ set(OPENACC_INSTALL_LIBDIR "lib${LLVM_LIBDIR_SUFFIX}/${OPENACC_TARGET_SUBDIR}" CACHE STRING
+ "Path where built OpenACC libraries should be installed.")
+else()
+ set(OPENACC_INSTALL_LIBDIR "lib${LLVM_LIBDIR_SUFFIX}" CACHE STRING
+ "Path where built OpenACC libraries should be installed.")
+endif()
+
+# Use the current compiler target to determine the appropriate runtime to build.
+if("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx|^spirv64" OR
+ "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^amdgcn|^nvptx|^spirv64")
+ add_subdirectory(device)
+endif()
diff --git a/openacc/device/CMakeLists.txt b/openacc/device/CMakeLists.txt
new file mode 100644
index 00000000000000..4b18fcb21013ae
--- /dev/null
+++ b/openacc/device/CMakeLists.txt
@@ -0,0 +1,82 @@
+# Ensure the compiler is a valid clang when building the GPU target.
+set(req_ver "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}")
+if(LLVM_VERSION_MAJOR AND NOT (CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang" AND
+ ${CMAKE_CXX_COMPILER_VERSION} VERSION_EQUAL "${req_ver}"))
+ message(FATAL_ERROR "Cannot build OpenACC GPU device runtime. CMake compiler "
+ "'${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}' "
+ " is not 'Clang ${req_ver}'.")
+endif()
+
+set(src_files
+ ${CMAKE_CURRENT_SOURCE_DIR}/src/AccEntryImpl.cpp
+)
+
+list(APPEND compile_options -flto)
+list(APPEND compile_options -fvisibility=hidden)
+list(APPEND compile_options -nogpulib)
+list(APPEND compile_options -nostdlibinc)
+list(APPEND compile_options -fno-rtti)
+list(APPEND compile_options -fno-exceptions)
+list(APPEND compile_options -fconvergent-functions)
+list(APPEND compile_options -Wno-unknown-cuda-version)
+
+if(LLVM_DEFAULT_TARGET_TRIPLE)
+ list(APPEND compile_options --target=${LLVM_DEFAULT_TARGET_TRIPLE})
+endif()
+
+list(APPEND compile_options "SHELL: -mllvm -vectorize-slp=false")
+if("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn" OR
+ "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^amdgcn")
+ set(target_name "amdgpu")
+ list(APPEND compile_options "SHELL:-Xclang -mcode-object-version=none")
+elseif("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^nvptx" OR
+ "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^nvptx")
+ set(target_name "nvptx")
+ list(APPEND compile_options --cuda-feature=+ptx63)
+elseif("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^spirv" OR
+ "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^spirv")
+ set(target_name "spirv")
+endif()
+
+# Produce a bitcode file via the linker's LTO pass.
+add_executable(libllvmaccdevice ${src_files})
+set_target_properties(libllvmaccdevice PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY "${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}"
+ LINKER_LANGUAGE CXX
+ BUILD_RPATH ""
+ INSTALL_RPATH ""
+ RUNTIME_OUTPUT_NAME libacctarget-${target_name}.bc)
+
+target_include_directories(libllvmaccdevice PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../offload/libacctarget/include
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../offload/include)
+target_compile_options(libllvmaccdevice PRIVATE ${compile_options})
+if(NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^spirv" AND
+ NOT "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^spirv")
+ target_link_options(libllvmaccdevice PRIVATE
+ "-flto" "-r" "-nostdlib" "-Wl,--lto-emit-llvm")
+else()
+ target_link_options(libllvmaccdevice PRIVATE
+ "-nostdlib" "-emit-llvm")
+endif()
+
+if(LLVM_DEFAULT_TARGET_TRIPLE)
+ target_link_options(libllvmaccdevice PRIVATE "--target=${LLVM_DEFAULT_TARGET_TRIPLE}")
+endif()
+install(TARGETS libllvmaccdevice
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ
+ DESTINATION ${OPENACC_INSTALL_LIBDIR})
+
+add_library(llvmaccdevice.all_objs OBJECT IMPORTED)
+set_property(TARGET llvmaccdevice.all_objs APPEND PROPERTY IMPORTED_OBJECTS
+ ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}/libacctarget-${target_name}.bc)
+
+add_library(llvmaccdevice STATIC)
+add_dependencies(llvmaccdevice libllvmaccdevice)
+set_target_properties(llvmaccdevice PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}"
+ LINKER_LANGUAGE CXX
+)
+target_link_libraries(llvmaccdevice PRIVATE llvmaccdevice.all_objs)
+install(TARGETS llvmaccdevice ARCHIVE DESTINATION "${OPENACC_INSTALL_LIBDIR}")
diff --git a/openacc/device/src/AccEntryImpl.cpp b/openacc/device/src/AccEntryImpl.cpp
new file mode 100644
index 00000000000000..7bd9d29d150ee8
--- /dev/null
+++ b/openacc/device/src/AccEntryImpl.cpp
@@ -0,0 +1,23 @@
+//===- AccEntryImpl.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "openacc.h"
+
+extern "C" {
+#ifdef __CUDA_ARCH__
+#pragma omp declare target
+int acc_on_device(acc_device_t DevType) {
+ if (DevType == acc_device_not_host || DevType == acc_device_nvidia) {
+ return true;
+ }
+ return false;
+}
+int acc_on_device_(acc_device_t *DevType) { return acc_on_device(*DevType); }
+#pragma omp end declare target
+#endif
+}
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index a0ec44617c9c77..82cf318312be7e 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -44,7 +44,7 @@ list(INSERT CMAKE_MODULE_PATH 0
# compiler-rt can depend on libc++, so we put it after. Offload can depend on
# flang-rt so we order it before.
set(LLVM_DEFAULT_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;libclc;openmp;offload")
-set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;libclc;openmp;flang-rt;offload;llvm-libgcc;libsycl;orc-rt")
+set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;libclc;openmp;openacc;flang-rt;offload;llvm-libgcc;libsycl;orc-rt")
set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
"Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
if(LLVM_ENABLE_RUNTIMES STREQUAL "all" )
More information about the cfe-commits
mailing list