[llvm-branch-commits] [llvm] [Offload][Omp] Use olMemCpy3D in omp_target_memcpy_rect (PR #241960)
Alex Duran via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 12 08:46:27 PST 2026
https://github.com/adurang created https://github.com/llvm/llvm-project/pull/241960
Implement olMemCpyRect more generically in DeviceTy using the
accelerated olMemCy3D if supported by the device.
Use it to reimplement omp_target_memcpy_rect.
Assisted by Claude.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 3b6fc68b9cd809a37f9cbe005443b77cb811a834 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Thu, 12 Nov 2026 08:43:27 -0800
Subject: [PATCH] [Offload][Omp] Use olMemCpy3D in omp_target_memcpy_rect
Implement olMemCpyRect more generically in DeviceTy using the
accelerated olMemCy3D if supported by the device.
Use it to reimplement omp_target_memcpy_rect.
Assisted by Claude.
---
offload/include/device.h | 51 ++++
offload/libompaccsupport/device.cpp | 285 +++++++++++++++++++
offload/libomptarget/OpenMP/API.cpp | 68 +++--
offload/test/api/omp_target_memcpy_rect_3d.c | 124 ++++++++
4 files changed, 500 insertions(+), 28 deletions(-)
create mode 100644 offload/test/api/omp_target_memcpy_rect_3d.c
diff --git a/offload/include/device.h b/offload/include/device.h
index 14b16387a4bb6..6541f921da559 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -13,6 +13,7 @@
#ifndef _OMPTARGET_DEVICE_H
#define _OMPTARGET_DEVICE_H
+#include <atomic>
#include <cassert>
#include <cstddef>
#include <cstdint>
@@ -32,6 +33,7 @@
#include "OpenMP/Mapping.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -117,6 +119,22 @@ struct RecordReplayTy;
} // namespace llvm::omp::target::rr
using RecordReplayTy = llvm::omp::target::rr::RecordReplayTy;
+/// A rectangular copy; per-dimension arrays are ordered slowest-varying first.
+struct MemcpyRectTy {
+ size_t ElementSize;
+ int NumDims;
+ const size_t *Volume;
+ const size_t *DstOffsets;
+ const size_t *SrcOffsets;
+ const size_t *DstDimensions;
+ const size_t *SrcDimensions;
+};
+
+/// Call \p CopyRow for each contiguous row of \p Rect.
+llvm::Error forEachMemcpyRectRow(
+ void *Dst, const void *Src, const MemcpyRectTy &Rect,
+ llvm::function_ref<llvm::Error(void *, const void *, size_t)> CopyRow);
+
struct DeviceTy {
int32_t DeviceID;
ol_platform_handle_t Platform;
@@ -242,6 +260,17 @@ struct DeviceTy {
int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
int64_t Size, AsyncInfoTy &AsyncInfo);
+ /// Rectangular versions of submitData, retrieveData and dataExchange. A null
+ /// \p AsyncInfo makes the copy synchronous.
+ llvm::Error submitDataRect(void *TgtPtr, const void *HstPtr,
+ const MemcpyRectTy &Rect, AsyncInfoTy *AsyncInfo);
+ llvm::Error retrieveDataRect(void *HstPtr, const void *TgtPtr,
+ const MemcpyRectTy &Rect,
+ AsyncInfoTy *AsyncInfo);
+ llvm::Error dataExchangeRect(const void *SrcPtr, DeviceTy &DstDev,
+ void *DstPtr, const MemcpyRectTy &Rect,
+ AsyncInfoTy *AsyncInfo);
+
/// Register (and, if \p LockMemory, page-lock) the host buffer \p HstPtr
/// with \p Size bytes, returning the device-accessible pointer.
llvm::Expected<void *> registerMemory(void *HstPtr, int64_t Size,
@@ -372,6 +401,25 @@ struct DeviceTy {
}
private:
+ /// This device is the destination of HostToDevice and the source otherwise.
+ enum class CopyDirTy { HostToDevice, DeviceToHost, DeviceToDevice };
+
+ /// Single liboffload 2D/3D copy; \p PeerDev is the destination of a
+ /// DeviceToDevice copy. Returns false if another path is needed.
+ llvm::Expected<bool> dataMemcpy3D(CopyDirTy Dir, void *DstPtr,
+ const ol_memcpy3d_layout_t &DstLayout,
+ const void *SrcPtr,
+ const ol_memcpy3d_layout_t &SrcLayout,
+ ol_dimensions_t Size, DeviceTy *PeerDev,
+ AsyncInfoTy *AsyncInfo);
+
+ /// Implementation of the rectangular copies; \p CopyRow copies one row.
+ llvm::Error dataMemcpyRect(
+ CopyDirTy Dir, void *Dst, const void *Src, const MemcpyRectTy &Rect,
+ DeviceTy *PeerDev, AsyncInfoTy *AsyncInfo,
+ llvm::function_ref<int32_t(void *, const void *, size_t, AsyncInfoTy &)>
+ CopyRow);
+
/// Query the properties used to compute the launch geometry of a kernel.
llvm::Error initGeometryProps();
@@ -424,6 +472,9 @@ struct DeviceTy {
/// Indicate whether failures when locking mapped buffers should be ignored.
bool IgnoreLockMappedFailures = true;
+
+ /// Set once 2D/3D copies are found unsupported on this device.
+ std::atomic<bool> Memcpy3DUnsupported{false};
};
#endif
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index d67b27cad2c62..ed0eb9effdd62 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -36,6 +36,7 @@
#include <cstdio>
#include <limits>
#include <mutex>
+#include <optional>
#include <string>
#include <thread>
@@ -697,6 +698,290 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
return OFFLOAD_SUCCESS;
}
+// Returns false if another path is needed; \p Unsupported is set if liboffload
+// reported the copy as unsupported.
+static llvm::Expected<bool> memcpy3DImpl(
+ ol_queue_handle_t Queue, void *DstPtr, ol_device_handle_t DstHandle,
+ const ol_memcpy3d_layout_t &DstLayout, const void *SrcPtr,
+ ol_device_handle_t SrcHandle, const ol_memcpy3d_layout_t &SrcLayout,
+ ol_dimensions_t Size, bool &Unsupported) {
+ if (auto Res = olMemcpy3D(Queue, DstPtr, DstHandle, &DstLayout, SrcPtr,
+ SrcHandle, &SrcLayout, Size)) {
+ Unsupported = Res->Code == OL_ERRC_UNSUPPORTED;
+ // The generic path accepts layouts that liboffload rejects.
+ if (Unsupported || Res->Code == OL_ERRC_INVALID_SIZE)
+ return false;
+ return createError(ErrorCode::BackendFailure,
+ "failure in rectangular memcpy: %s", Res->Details);
+ }
+ return true;
+}
+
+// First byte of the region described by \p Layout.
+static char *regionStart(char *Base, const ol_memcpy3d_layout_t &Layout) {
+ return Base + Layout.Offset.x + Layout.Offset.y * Layout.Pitch +
+ Layout.Offset.z * Layout.SlicePitch;
+}
+
+llvm::Expected<bool> DeviceTy::dataMemcpy3D(
+ CopyDirTy Dir, void *DstPtr, const ol_memcpy3d_layout_t &DstLayout,
+ const void *SrcPtr, const ol_memcpy3d_layout_t &SrcLayout,
+ ol_dimensions_t Size, DeviceTy *PeerDev, AsyncInfoTy *AsyncInfo) {
+ assert((Dir != CopyDirTy::DeviceToDevice || PeerDev) &&
+ "DeviceToDevice needs a destination device");
+ ol_queue_handle_t Queue = nullptr;
+ if (AsyncInfo) {
+ Queue = AsyncInfo->getQueue();
+ if (!Queue)
+ return createError(ErrorCode::BackendFailure,
+ "no queue available for rectangular memcpy");
+ }
+ ol_device_handle_t DstHandle;
+ ol_device_handle_t SrcHandle;
+ switch (Dir) {
+ case CopyDirTy::HostToDevice:
+ DstHandle = DeviceHandle;
+ SrcHandle = nullptr;
+ break;
+ case CopyDirTy::DeviceToHost:
+ DstHandle = nullptr;
+ SrcHandle = DeviceHandle;
+ break;
+ case CopyDirTy::DeviceToDevice:
+ DstHandle = PeerDev->DeviceHandle;
+ SrcHandle = DeviceHandle;
+ break;
+ }
+
+ bool Unsupported = false;
+ auto Result = memcpy3DImpl(Queue, DstPtr, DstHandle, DstLayout, SrcPtr,
+ SrcHandle, SrcLayout, Size, Unsupported);
+ // Device-to-device may fail just because the devices cannot exchange data.
+ if (Unsupported && Dir != CopyDirTy::DeviceToDevice)
+ Memcpy3DUnsupported = true;
+ if (!Result || !*Result)
+ return Result;
+
+ // Report the copy as a single transfer, once it is known to be done this way.
+ void *DstStart = regionStart(static_cast<char *>(DstPtr), DstLayout);
+ void *SrcStart =
+ regionStart(static_cast<char *>(const_cast<void *>(SrcPtr)), SrcLayout);
+ int64_t Bytes = static_cast<int64_t>(Size.x) * Size.y * Size.z;
+ if (Dir != CopyDirTy::DeviceToDevice &&
+ (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)) {
+ bool H2D = Dir == CopyDirTy::HostToDevice;
+ MappingInfo.printCopyInfo(H2D ? DstStart : SrcStart,
+ H2D ? SrcStart : DstStart, Bytes, H2D,
+ /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr);
+ }
+
+ // Tool anchors around the transfer.
+ // clang-format messes the following block so we need to disable it.
+ // clang-format off
+ OMPT_IF_BUILT(
+ // Device-to-device is reported as from_device, like dataExchange.
+ auto Callbacks =
+ RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>();
+ int32_t SrcID;
+ int32_t DstID;
+ switch (Dir) {
+ case CopyDirTy::HostToDevice:
+ Callbacks =
+ RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>();
+ SrcID = omp_initial_device;
+ DstID = DeviceID;
+ break;
+ case CopyDirTy::DeviceToHost:
+ SrcID = DeviceID;
+ DstID = omp_initial_device;
+ break;
+ case CopyDirTy::DeviceToDevice:
+ SrcID = DeviceID;
+ DstID = PeerDev->DeviceID;
+ break;
+ }
+ InterfaceRAII TargetDataRAII(
+ Callbacks, SrcID, SrcStart, DstID, DstStart, Bytes,
+ /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
+ // clang-format on
+
+ return true;
+}
+
+llvm::Error forEachMemcpyRectRow(
+ void *Dst, const void *Src, const MemcpyRectTy &Rect,
+ llvm::function_ref<llvm::Error(void *, const void *, size_t)> CopyRow) {
+ const auto [ElementSize, NumDims, Volume, DstOffsets, SrcOffsets,
+ DstDimensions, SrcDimensions] = Rect;
+ for (int I = 0; I < NumDims; ++I)
+ if (!Volume[I])
+ return llvm::Error::success();
+
+ llvm::SmallVector<size_t, 4> DstStride(NumDims), SrcStride(NumDims);
+ DstStride[NumDims - 1] = SrcStride[NumDims - 1] = ElementSize;
+ for (int I = NumDims - 2; I >= 0; --I) {
+ DstStride[I] = DstStride[I + 1] * DstDimensions[I + 1];
+ SrcStride[I] = SrcStride[I + 1] * SrcDimensions[I + 1];
+ }
+ size_t DstBase = 0, SrcBase = 0;
+ for (int I = 0; I < NumDims; ++I) {
+ DstBase += DstOffsets[I] * DstStride[I];
+ SrcBase += SrcOffsets[I] * SrcStride[I];
+ }
+ size_t RowLength = Volume[NumDims - 1] * ElementSize;
+
+ llvm::SmallVector<size_t, 4> Idx(NumDims, 0);
+ while (true) {
+ char *DstAddr = static_cast<char *>(Dst) + DstBase;
+ const char *SrcAddr = static_cast<const char *>(Src) + SrcBase;
+ for (int I = 0; I < NumDims - 1; ++I) {
+ DstAddr += Idx[I] * DstStride[I];
+ SrcAddr += Idx[I] * SrcStride[I];
+ }
+ if (llvm::Error Err = CopyRow(DstAddr, SrcAddr, RowLength))
+ return Err;
+
+ int D = NumDims - 2;
+ for (; D >= 0; --D) {
+ if (++Idx[D] < Volume[D])
+ break;
+ Idx[D] = 0;
+ }
+ if (D < 0)
+ return llvm::Error::success();
+ }
+}
+
+// Describe \p Rect as a 3D copy, padding leading dimensions. Returns false if
+// the values do not fit what liboffload takes.
+static bool makeMemcpy3DArgs(const MemcpyRectTy &Rect,
+ ol_memcpy3d_layout_t &DstLayout,
+ ol_memcpy3d_layout_t &SrcLayout,
+ ol_dimensions_t &Size) {
+ size_t Vol[3] = {1, 1, 1}, DstOff[3] = {0, 0, 0}, SrcOff[3] = {0, 0, 0};
+ size_t DstDim[3] = {1, 1, 1}, SrcDim[3] = {1, 1, 1};
+ for (int I = 0; I < Rect.NumDims; ++I) {
+ int J = 3 - Rect.NumDims + I;
+ Vol[J] = Rect.Volume[I];
+ DstOff[J] = Rect.DstOffsets[I];
+ SrcOff[J] = Rect.SrcOffsets[I];
+ DstDim[J] = Rect.DstDimensions[I];
+ SrcDim[J] = Rect.SrcDimensions[I];
+ }
+
+ // Offsets and extents must fit in 32 bits; pitches must not overflow.
+ constexpr size_t Max = std::numeric_limits<uint32_t>::max();
+ bool InRange = true;
+ auto Mul = [&](size_t A, size_t B) {
+ size_t R;
+ InRange &= !__builtin_mul_overflow(A, B, &R);
+ return R;
+ };
+ size_t ES = Rect.ElementSize;
+ size_t DstPitch = Mul(DstDim[2], ES);
+ size_t SrcPitch = Mul(SrcDim[2], ES);
+ size_t DstSlicePitch = Mul(DstDim[1], DstPitch);
+ size_t SrcSlicePitch = Mul(SrcDim[1], SrcPitch);
+ size_t Width = Mul(Vol[2], ES);
+ size_t DstX = Mul(DstOff[2], ES);
+ size_t SrcX = Mul(SrcOff[2], ES);
+ for (size_t V : {Width, Vol[1], Vol[0], DstX, DstOff[1], DstOff[0], SrcX,
+ SrcOff[1], SrcOff[0]})
+ InRange &= V <= Max;
+ if (!InRange)
+ return false;
+
+ DstLayout = {{static_cast<uint32_t>(DstX), static_cast<uint32_t>(DstOff[1]),
+ static_cast<uint32_t>(DstOff[0])},
+ DstPitch,
+ DstSlicePitch};
+ SrcLayout = {{static_cast<uint32_t>(SrcX), static_cast<uint32_t>(SrcOff[1]),
+ static_cast<uint32_t>(SrcOff[0])},
+ SrcPitch,
+ SrcSlicePitch};
+ Size = {static_cast<uint32_t>(Width), static_cast<uint32_t>(Vol[1]),
+ static_cast<uint32_t>(Vol[0])};
+ return true;
+}
+
+llvm::Error DeviceTy::dataMemcpyRect(
+ CopyDirTy Dir, void *Dst, const void *Src, const MemcpyRectTy &Rect,
+ DeviceTy *PeerDev, AsyncInfoTy *AsyncInfo,
+ llvm::function_ref<int32_t(void *, const void *, size_t, AsyncInfoTy &)>
+ CopyRow) {
+ // Fast path: one liboffload call.
+ ol_memcpy3d_layout_t DstLayout, SrcLayout;
+ ol_dimensions_t Size;
+ if (!Memcpy3DUnsupported && Rect.NumDims >= 2 && Rect.NumDims <= 3 &&
+ makeMemcpy3DArgs(Rect, DstLayout, SrcLayout, Size)) {
+ auto SupportedOrErr = dataMemcpy3D(Dir, Dst, DstLayout, Src, SrcLayout,
+ Size, PeerDev, AsyncInfo);
+ if (!SupportedOrErr)
+ return SupportedOrErr.takeError();
+ if (*SupportedOrErr)
+ return llvm::Error::success();
+ }
+
+ // Generic path: one copy per contiguous row.
+ // Without an AsyncInfo, use a local one and synchronize at the end.
+ std::optional<AsyncInfoTy> LocalAsyncInfo;
+ AsyncInfoTy *Info = AsyncInfo;
+ if (!Info) {
+ LocalAsyncInfo.emplace(*this);
+ Info = &*LocalAsyncInfo;
+ }
+
+ auto CopyCheckedRow = [&](void *DstAddr, const void *SrcAddr,
+ size_t Length) -> llvm::Error {
+ if (CopyRow(DstAddr, SrcAddr, Length, *Info) != OFFLOAD_SUCCESS)
+ return createError(ErrorCode::BackendFailure,
+ "failure in rectangular memcpy row copy: dst=%p "
+ "src=%p size=%zu",
+ DstAddr, SrcAddr, Length);
+ return llvm::Error::success();
+ };
+ if (llvm::Error Err = forEachMemcpyRectRow(Dst, Src, Rect, CopyCheckedRow))
+ return Err;
+
+ if (LocalAsyncInfo && LocalAsyncInfo->synchronize() != OFFLOAD_SUCCESS)
+ return createError(ErrorCode::BackendFailure,
+ "failed to synchronize rectangular memcpy");
+ return llvm::Error::success();
+}
+
+llvm::Error DeviceTy::submitDataRect(void *TgtPtr, const void *HstPtr,
+ const MemcpyRectTy &Rect,
+ AsyncInfoTy *AsyncInfo) {
+ auto CopyRow = [&](void *Dst, const void *Src, size_t Length,
+ AsyncInfoTy &Info) {
+ return submitData(Dst, const_cast<void *>(Src), Length, Info);
+ };
+ return dataMemcpyRect(CopyDirTy::HostToDevice, TgtPtr, HstPtr, Rect, nullptr,
+ AsyncInfo, CopyRow);
+}
+
+llvm::Error DeviceTy::retrieveDataRect(void *HstPtr, const void *TgtPtr,
+ const MemcpyRectTy &Rect,
+ AsyncInfoTy *AsyncInfo) {
+ auto CopyRow = [&](void *Dst, const void *Src, size_t Length,
+ AsyncInfoTy &Info) {
+ return retrieveData(Dst, const_cast<void *>(Src), Length, Info);
+ };
+ return dataMemcpyRect(CopyDirTy::DeviceToHost, HstPtr, TgtPtr, Rect, nullptr,
+ AsyncInfo, CopyRow);
+}
+
+llvm::Error DeviceTy::dataExchangeRect(const void *SrcPtr, DeviceTy &DstDev,
+ void *DstPtr, const MemcpyRectTy &Rect,
+ AsyncInfoTy *AsyncInfo) {
+ auto CopyRow = [&](void *Dst, const void *Src, size_t Length,
+ AsyncInfoTy &Info) {
+ return dataExchange(const_cast<void *>(Src), DstDev, Dst, Length, Info);
+ };
+ return dataMemcpyRect(CopyDirTy::DeviceToDevice, DstPtr, SrcPtr, Rect,
+ &DstDev, AsyncInfo, CopyRow);
+}
+
llvm::Expected<void *> DeviceTy::registerMemory(void *HstPtr, int64_t Size,
bool LockMemory) {
void *LockedPtr = nullptr;
diff --git a/offload/libomptarget/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp
index 742f9bae927b3..f3848d294ab0f 100644
--- a/offload/libomptarget/OpenMP/API.cpp
+++ b/offload/libomptarget/OpenMP/API.cpp
@@ -42,7 +42,6 @@ using namespace llvm::omp::target::ompt;
#endif
using namespace llvm::omp::target::debug;
-
void *targetAllocExplicit(size_t Size, int DeviceNum, int Kind,
const char *Name);
void targetFreeExplicit(void *DevicePtr, int DeviceNum, int Kind,
@@ -545,6 +544,16 @@ EXTERN int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length,
return Rc;
}
+// Returns the device for \p DeviceNum, or nullptr if it is the host.
+static DeviceTy *getDeviceOrHost(int DeviceNum) {
+ if (isInitialDevice(DeviceNum))
+ return nullptr;
+ auto DevOrErr = PM->getDevice(DeviceNum);
+ if (!DevOrErr)
+ FATAL_MESSAGE(DeviceNum, "%s", toString(DevOrErr.takeError()).c_str());
+ return &*DevOrErr;
+}
+
EXTERN int
omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
int NumDims, const size_t *Volume,
@@ -572,36 +581,39 @@ omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
return OFFLOAD_FAIL;
}
- int Rc;
- if (NumDims == 1) {
- Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0],
- ElementSize * DstOffsets[0],
- ElementSize * SrcOffsets[0], DstDevice, SrcDevice);
- } else {
- size_t DstSliceSize = ElementSize;
- size_t SrcSliceSize = ElementSize;
- for (int I = 1; I < NumDims; ++I) {
- DstSliceSize *= DstDimensions[I];
- SrcSliceSize *= SrcDimensions[I];
- }
-
- size_t DstOff = DstOffsets[0] * DstSliceSize;
- size_t SrcOff = SrcOffsets[0] * SrcSliceSize;
- for (size_t I = 0; I < Volume[0]; ++I) {
- Rc = omp_target_memcpy_rect(
- (char *)Dst + DstOff + DstSliceSize * I,
- (char *)const_cast<void *>(Src) + SrcOff + SrcSliceSize * I,
- ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1,
- DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice);
-
- if (Rc) {
- ODBG(ODT_Interface)
- << "Recursive call to " << __func__ << " returns unsuccessfully";
- return Rc;
- }
+ for (int I = 0; I < NumDims; ++I) {
+ if (!Volume[I]) {
+ ODBG(ODT_Interface) << "Call to " << __func__
+ << " with zero volume, nothing to do";
+ return OFFLOAD_SUCCESS;
}
}
+ DeviceTy *DstDev = getDeviceOrHost(DstDevice);
+ DeviceTy *SrcDev = getDeviceOrHost(SrcDevice);
+
+ MemcpyRectTy Rect = {ElementSize, NumDims, Volume, DstOffsets,
+ SrcOffsets, DstDimensions, SrcDimensions};
+ llvm::Error Err = [&]() -> llvm::Error {
+ if (!DstDev && !SrcDev)
+ return forEachMemcpyRectRow(
+ Dst, Src, Rect,
+ [](void *DstAddr, const void *SrcAddr, size_t Length) {
+ memcpy(DstAddr, SrcAddr, Length);
+ return llvm::Error::success();
+ });
+ if (!SrcDev)
+ return DstDev->submitDataRect(Dst, Src, Rect, nullptr);
+ if (!DstDev)
+ return SrcDev->retrieveDataRect(Dst, Src, Rect, nullptr);
+ return SrcDev->dataExchangeRect(Src, *DstDev, Dst, Rect, nullptr);
+ }();
+
+ int Rc = OFFLOAD_SUCCESS;
+ if (Err) {
+ REPORT() << toString(std::move(Err));
+ Rc = OFFLOAD_FAIL;
+ }
ODBG(ODT_Interface) << " returns " << Rc;
return Rc;
}
diff --git a/offload/test/api/omp_target_memcpy_rect_3d.c b/offload/test/api/omp_target_memcpy_rect_3d.c
new file mode 100644
index 0000000000000..e7a6a8c6a5413
--- /dev/null
+++ b/offload/test/api/omp_target_memcpy_rect_3d.c
@@ -0,0 +1,124 @@
+// RUN: %libomptarget-compile-and-run-generic | %fcheck-generic
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Reference implementation on host memory.
+static void ref_rect(char *dst, const char *src, size_t es, int nd,
+ const size_t *vol, const size_t *doff, const size_t *soff,
+ const size_t *ddim, const size_t *sdim) {
+ if (nd == 1) {
+ memcpy(dst + doff[0] * es, src + soff[0] * es, vol[0] * es);
+ return;
+ }
+ size_t dslice = es, sslice = es;
+ for (int i = 1; i < nd; ++i) {
+ dslice *= ddim[i];
+ sslice *= sdim[i];
+ }
+ for (size_t i = 0; i < vol[0]; ++i)
+ ref_rect(dst + (doff[0] + i) * dslice, src + (soff[0] + i) * sslice, es,
+ nd - 1, vol + 1, doff + 1, soff + 1, ddim + 1, sdim + 1);
+}
+
+enum Dir { H2H, H2D, D2H, D2D };
+
+static size_t total(int nd, const size_t *dim, size_t es) {
+ size_t n = es;
+ for (int i = 0; i < nd; ++i)
+ n *= dim[i];
+ return n;
+}
+
+// Run one copy in direction \p dir and compare against the reference.
+static int run_case(const char *name, enum Dir dir, int nd, size_t es,
+ const size_t *vol, const size_t *doff, const size_t *soff,
+ const size_t *ddim, const size_t *sdim) {
+ int host = omp_get_initial_device();
+ int dev = omp_get_default_device();
+ if (dev < 0 || dev >= omp_get_num_devices())
+ dev = host;
+
+ size_t sbytes = total(nd, sdim, es), dbytes = total(nd, ddim, es);
+ char *hsrc = malloc(sbytes), *href = calloc(1, dbytes),
+ *hout = calloc(1, dbytes);
+ for (size_t i = 0; i < sbytes; ++i)
+ hsrc[i] = (char)(i * 7 + 1);
+ ref_rect(href, hsrc, es, nd, vol, doff, soff, ddim, sdim);
+
+ int sdev = (dir == H2H || dir == H2D) ? host : dev;
+ int ddev = (dir == H2H || dir == D2H) ? host : dev;
+ char *src = sdev == host ? hsrc : omp_target_alloc(sbytes, sdev);
+ char *dst = ddev == host ? hout : omp_target_alloc(dbytes, ddev);
+ if (!src || !dst)
+ return 1;
+ if (sdev != host && omp_target_memcpy(src, hsrc, sbytes, 0, 0, sdev, host))
+ return 1;
+ if (ddev != host && omp_target_memcpy(dst, hout, dbytes, 0, 0, ddev, host))
+ return 1;
+
+ if (omp_target_memcpy_rect(dst, src, es, nd, vol, doff, soff, ddim, sdim,
+ ddev, sdev)) {
+ printf("%s: rect copy failed\n", name);
+ return 1;
+ }
+ if (ddev != host && omp_target_memcpy(hout, dst, dbytes, 0, 0, host, ddev))
+ return 1;
+
+ int rc = memcmp(href, hout, dbytes) != 0;
+ if (rc)
+ printf("%s: mismatch\n", name);
+ if (sdev != host)
+ omp_target_free(src, sdev);
+ if (ddev != host)
+ omp_target_free(dst, ddev);
+ free(hsrc);
+ free(href);
+ free(hout);
+ return rc;
+}
+
+int main() {
+ // 1D
+ size_t v1[1] = {10}, o1[1] = {3}, s1[1] = {5}, d1[1] = {32};
+ // 2D: 3 rows of 5 elements
+ size_t v2[2] = {3, 5}, so2[2] = {1, 2}, do2[2] = {2, 1};
+ size_t sd2[2] = {6, 9}, dd2[2] = {7, 10};
+ // 3D
+ size_t v3[3] = {2, 3, 4}, so3[3] = {1, 2, 3}, do3[3] = {2, 1, 0};
+ size_t dim3[3] = {4, 6, 8};
+ // 4D (handled by the generic per-row path)
+ size_t v4[4] = {2, 2, 3, 4}, so4[4] = {1, 0, 1, 2}, do4[4] = {0, 1, 1, 1};
+ size_t dim4[4] = {3, 3, 4, 7};
+
+ static const char *dirs[] = {"H2H", "H2D", "D2H", "D2D"};
+ char name[64];
+ for (int d = H2H; d <= D2D; ++d) {
+ for (size_t es = 4; es <= 8; es += 4) {
+#define CASE(N, V, DO, SO, DD, SD) \
+ snprintf(name, sizeof(name), "%dD %s es=%zu", N, dirs[d], es); \
+ if (run_case(name, d, N, es, V, DO, SO, DD, SD)) \
+ return 1;
+ CASE(1, v1, o1, s1, d1, d1)
+ CASE(2, v2, do2, so2, dd2, sd2)
+ CASE(3, v3, do3, so3, dim3, dim3)
+ CASE(4, v4, do4, so4, dim4, dim4)
+#undef CASE
+ }
+ }
+
+ // Nothing to copy: succeeds without looking at the devices.
+ size_t vz[2] = {0, 4}, oz[2] = {0, 0}, dz[2] = {4, 4};
+ char a, b;
+ if (omp_target_memcpy_rect(&a, &b, 1, 2, vz, oz, oz, dz, dz, 12345, 12345)) {
+ printf("zero volume failed\n");
+ return 1;
+ }
+
+ printf("PASS\n");
+ return 0;
+}
+
+// CHECK: PASS
More information about the llvm-branch-commits
mailing list