[llvm-branch-commits] [llvm] [Offload][Omp] Use olMemCpy3D in omp_target_memcpy_rect (PR #241960)

Alex Duran via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Thu Nov 12 08:46:27 PST 2026


https://github.com/adurang created https://github.com/llvm/llvm-project/pull/241960

Implement olMemCpyRect more generically in DeviceTy using the
accelerated olMemCy3D if supported by the device.

Use it to reimplement omp_target_memcpy_rect.

Assisted by Claude.

---

<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>

>From 3b6fc68b9cd809a37f9cbe005443b77cb811a834 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Thu, 12 Nov 2026 08:43:27 -0800
Subject: [PATCH] [Offload][Omp] Use olMemCpy3D in omp_target_memcpy_rect

Implement olMemCpyRect more generically in DeviceTy using the
accelerated olMemCy3D if supported by the device.

Use it to reimplement omp_target_memcpy_rect.

Assisted by Claude.
---
 offload/include/device.h                     |  51 ++++
 offload/libompaccsupport/device.cpp          | 285 +++++++++++++++++++
 offload/libomptarget/OpenMP/API.cpp          |  68 +++--
 offload/test/api/omp_target_memcpy_rect_3d.c | 124 ++++++++
 4 files changed, 500 insertions(+), 28 deletions(-)
 create mode 100644 offload/test/api/omp_target_memcpy_rect_3d.c

diff --git a/offload/include/device.h b/offload/include/device.h
index 14b16387a4bb6..6541f921da559 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -13,6 +13,7 @@
 #ifndef _OMPTARGET_DEVICE_H
 #define _OMPTARGET_DEVICE_H
 
+#include <atomic>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -32,6 +33,7 @@
 #include "OpenMP/Mapping.h"
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -117,6 +119,22 @@ struct RecordReplayTy;
 } // namespace llvm::omp::target::rr
 using RecordReplayTy = llvm::omp::target::rr::RecordReplayTy;
 
+/// A rectangular copy; per-dimension arrays are ordered slowest-varying first.
+struct MemcpyRectTy {
+  size_t ElementSize;
+  int NumDims;
+  const size_t *Volume;
+  const size_t *DstOffsets;
+  const size_t *SrcOffsets;
+  const size_t *DstDimensions;
+  const size_t *SrcDimensions;
+};
+
+/// Call \p CopyRow for each contiguous row of \p Rect.
+llvm::Error forEachMemcpyRectRow(
+    void *Dst, const void *Src, const MemcpyRectTy &Rect,
+    llvm::function_ref<llvm::Error(void *, const void *, size_t)> CopyRow);
+
 struct DeviceTy {
   int32_t DeviceID;
   ol_platform_handle_t Platform;
@@ -242,6 +260,17 @@ struct DeviceTy {
   int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
                        int64_t Size, AsyncInfoTy &AsyncInfo);
 
+  /// Rectangular versions of submitData, retrieveData and dataExchange. A null
+  /// \p AsyncInfo makes the copy synchronous.
+  llvm::Error submitDataRect(void *TgtPtr, const void *HstPtr,
+                             const MemcpyRectTy &Rect, AsyncInfoTy *AsyncInfo);
+  llvm::Error retrieveDataRect(void *HstPtr, const void *TgtPtr,
+                               const MemcpyRectTy &Rect,
+                               AsyncInfoTy *AsyncInfo);
+  llvm::Error dataExchangeRect(const void *SrcPtr, DeviceTy &DstDev,
+                               void *DstPtr, const MemcpyRectTy &Rect,
+                               AsyncInfoTy *AsyncInfo);
+
   /// Register (and, if \p LockMemory, page-lock) the host buffer \p HstPtr
   /// with \p Size bytes, returning the device-accessible pointer.
   llvm::Expected<void *> registerMemory(void *HstPtr, int64_t Size,
@@ -372,6 +401,25 @@ struct DeviceTy {
   }
 
 private:
+  /// This device is the destination of HostToDevice and the source otherwise.
+  enum class CopyDirTy { HostToDevice, DeviceToHost, DeviceToDevice };
+
+  /// Single liboffload 2D/3D copy; \p PeerDev is the destination of a
+  /// DeviceToDevice copy. Returns false if another path is needed.
+  llvm::Expected<bool> dataMemcpy3D(CopyDirTy Dir, void *DstPtr,
+                                    const ol_memcpy3d_layout_t &DstLayout,
+                                    const void *SrcPtr,
+                                    const ol_memcpy3d_layout_t &SrcLayout,
+                                    ol_dimensions_t Size, DeviceTy *PeerDev,
+                                    AsyncInfoTy *AsyncInfo);
+
+  /// Implementation of the rectangular copies; \p CopyRow copies one row.
+  llvm::Error dataMemcpyRect(
+      CopyDirTy Dir, void *Dst, const void *Src, const MemcpyRectTy &Rect,
+      DeviceTy *PeerDev, AsyncInfoTy *AsyncInfo,
+      llvm::function_ref<int32_t(void *, const void *, size_t, AsyncInfoTy &)>
+          CopyRow);
+
   /// Query the properties used to compute the launch geometry of a kernel.
   llvm::Error initGeometryProps();
 
@@ -424,6 +472,9 @@ struct DeviceTy {
 
   /// Indicate whether failures when locking mapped buffers should be ignored.
   bool IgnoreLockMappedFailures = true;
+
+  /// Set once 2D/3D copies are found unsupported on this device.
+  std::atomic<bool> Memcpy3DUnsupported{false};
 };
 
 #endif
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index d67b27cad2c62..ed0eb9effdd62 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -36,6 +36,7 @@
 #include <cstdio>
 #include <limits>
 #include <mutex>
+#include <optional>
 #include <string>
 #include <thread>
 
@@ -697,6 +698,290 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
   return OFFLOAD_SUCCESS;
 }
 
+// Returns false if another path is needed; \p Unsupported is set if liboffload
+// reported the copy as unsupported.
+static llvm::Expected<bool> memcpy3DImpl(
+    ol_queue_handle_t Queue, void *DstPtr, ol_device_handle_t DstHandle,
+    const ol_memcpy3d_layout_t &DstLayout, const void *SrcPtr,
+    ol_device_handle_t SrcHandle, const ol_memcpy3d_layout_t &SrcLayout,
+    ol_dimensions_t Size, bool &Unsupported) {
+  if (auto Res = olMemcpy3D(Queue, DstPtr, DstHandle, &DstLayout, SrcPtr,
+                            SrcHandle, &SrcLayout, Size)) {
+    Unsupported = Res->Code == OL_ERRC_UNSUPPORTED;
+    // The generic path accepts layouts that liboffload rejects.
+    if (Unsupported || Res->Code == OL_ERRC_INVALID_SIZE)
+      return false;
+    return createError(ErrorCode::BackendFailure,
+                       "failure in rectangular memcpy: %s", Res->Details);
+  }
+  return true;
+}
+
+// First byte of the region described by \p Layout.
+static char *regionStart(char *Base, const ol_memcpy3d_layout_t &Layout) {
+  return Base + Layout.Offset.x + Layout.Offset.y * Layout.Pitch +
+         Layout.Offset.z * Layout.SlicePitch;
+}
+
+llvm::Expected<bool> DeviceTy::dataMemcpy3D(
+    CopyDirTy Dir, void *DstPtr, const ol_memcpy3d_layout_t &DstLayout,
+    const void *SrcPtr, const ol_memcpy3d_layout_t &SrcLayout,
+    ol_dimensions_t Size, DeviceTy *PeerDev, AsyncInfoTy *AsyncInfo) {
+  assert((Dir != CopyDirTy::DeviceToDevice || PeerDev) &&
+         "DeviceToDevice needs a destination device");
+  ol_queue_handle_t Queue = nullptr;
+  if (AsyncInfo) {
+    Queue = AsyncInfo->getQueue();
+    if (!Queue)
+      return createError(ErrorCode::BackendFailure,
+                         "no queue available for rectangular memcpy");
+  }
+  ol_device_handle_t DstHandle;
+  ol_device_handle_t SrcHandle;
+  switch (Dir) {
+  case CopyDirTy::HostToDevice:
+    DstHandle = DeviceHandle;
+    SrcHandle = nullptr;
+    break;
+  case CopyDirTy::DeviceToHost:
+    DstHandle = nullptr;
+    SrcHandle = DeviceHandle;
+    break;
+  case CopyDirTy::DeviceToDevice:
+    DstHandle = PeerDev->DeviceHandle;
+    SrcHandle = DeviceHandle;
+    break;
+  }
+
+  bool Unsupported = false;
+  auto Result = memcpy3DImpl(Queue, DstPtr, DstHandle, DstLayout, SrcPtr,
+                             SrcHandle, SrcLayout, Size, Unsupported);
+  // Device-to-device may fail just because the devices cannot exchange data.
+  if (Unsupported && Dir != CopyDirTy::DeviceToDevice)
+    Memcpy3DUnsupported = true;
+  if (!Result || !*Result)
+    return Result;
+
+  // Report the copy as a single transfer, once it is known to be done this way.
+  void *DstStart = regionStart(static_cast<char *>(DstPtr), DstLayout);
+  void *SrcStart =
+      regionStart(static_cast<char *>(const_cast<void *>(SrcPtr)), SrcLayout);
+  int64_t Bytes = static_cast<int64_t>(Size.x) * Size.y * Size.z;
+  if (Dir != CopyDirTy::DeviceToDevice &&
+      (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)) {
+    bool H2D = Dir == CopyDirTy::HostToDevice;
+    MappingInfo.printCopyInfo(H2D ? DstStart : SrcStart,
+                              H2D ? SrcStart : DstStart, Bytes, H2D,
+                              /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr);
+  }
+
+  // Tool anchors around the transfer.
+  // clang-format messes the following block so we need to disable it.
+  // clang-format off
+  OMPT_IF_BUILT(
+      // Device-to-device is reported as from_device, like dataExchange.
+      auto Callbacks =
+          RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>();
+      int32_t SrcID;
+      int32_t DstID;
+      switch (Dir) {
+      case CopyDirTy::HostToDevice:
+        Callbacks =
+            RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>();
+        SrcID = omp_initial_device;
+        DstID = DeviceID;
+        break;
+      case CopyDirTy::DeviceToHost:
+        SrcID = DeviceID;
+        DstID = omp_initial_device;
+        break;
+      case CopyDirTy::DeviceToDevice:
+        SrcID = DeviceID;
+        DstID = PeerDev->DeviceID;
+        break;
+      }
+      InterfaceRAII TargetDataRAII(
+          Callbacks, SrcID, SrcStart, DstID, DstStart, Bytes,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
+  // clang-format on
+
+  return true;
+}
+
+llvm::Error forEachMemcpyRectRow(
+    void *Dst, const void *Src, const MemcpyRectTy &Rect,
+    llvm::function_ref<llvm::Error(void *, const void *, size_t)> CopyRow) {
+  const auto [ElementSize, NumDims, Volume, DstOffsets, SrcOffsets,
+              DstDimensions, SrcDimensions] = Rect;
+  for (int I = 0; I < NumDims; ++I)
+    if (!Volume[I])
+      return llvm::Error::success();
+
+  llvm::SmallVector<size_t, 4> DstStride(NumDims), SrcStride(NumDims);
+  DstStride[NumDims - 1] = SrcStride[NumDims - 1] = ElementSize;
+  for (int I = NumDims - 2; I >= 0; --I) {
+    DstStride[I] = DstStride[I + 1] * DstDimensions[I + 1];
+    SrcStride[I] = SrcStride[I + 1] * SrcDimensions[I + 1];
+  }
+  size_t DstBase = 0, SrcBase = 0;
+  for (int I = 0; I < NumDims; ++I) {
+    DstBase += DstOffsets[I] * DstStride[I];
+    SrcBase += SrcOffsets[I] * SrcStride[I];
+  }
+  size_t RowLength = Volume[NumDims - 1] * ElementSize;
+
+  llvm::SmallVector<size_t, 4> Idx(NumDims, 0);
+  while (true) {
+    char *DstAddr = static_cast<char *>(Dst) + DstBase;
+    const char *SrcAddr = static_cast<const char *>(Src) + SrcBase;
+    for (int I = 0; I < NumDims - 1; ++I) {
+      DstAddr += Idx[I] * DstStride[I];
+      SrcAddr += Idx[I] * SrcStride[I];
+    }
+    if (llvm::Error Err = CopyRow(DstAddr, SrcAddr, RowLength))
+      return Err;
+
+    int D = NumDims - 2;
+    for (; D >= 0; --D) {
+      if (++Idx[D] < Volume[D])
+        break;
+      Idx[D] = 0;
+    }
+    if (D < 0)
+      return llvm::Error::success();
+  }
+}
+
+// Describe \p Rect as a 3D copy, padding leading dimensions. Returns false if
+// the values do not fit what liboffload takes.
+static bool makeMemcpy3DArgs(const MemcpyRectTy &Rect,
+                             ol_memcpy3d_layout_t &DstLayout,
+                             ol_memcpy3d_layout_t &SrcLayout,
+                             ol_dimensions_t &Size) {
+  size_t Vol[3] = {1, 1, 1}, DstOff[3] = {0, 0, 0}, SrcOff[3] = {0, 0, 0};
+  size_t DstDim[3] = {1, 1, 1}, SrcDim[3] = {1, 1, 1};
+  for (int I = 0; I < Rect.NumDims; ++I) {
+    int J = 3 - Rect.NumDims + I;
+    Vol[J] = Rect.Volume[I];
+    DstOff[J] = Rect.DstOffsets[I];
+    SrcOff[J] = Rect.SrcOffsets[I];
+    DstDim[J] = Rect.DstDimensions[I];
+    SrcDim[J] = Rect.SrcDimensions[I];
+  }
+
+  // Offsets and extents must fit in 32 bits; pitches must not overflow.
+  constexpr size_t Max = std::numeric_limits<uint32_t>::max();
+  bool InRange = true;
+  auto Mul = [&](size_t A, size_t B) {
+    size_t R;
+    InRange &= !__builtin_mul_overflow(A, B, &R);
+    return R;
+  };
+  size_t ES = Rect.ElementSize;
+  size_t DstPitch = Mul(DstDim[2], ES);
+  size_t SrcPitch = Mul(SrcDim[2], ES);
+  size_t DstSlicePitch = Mul(DstDim[1], DstPitch);
+  size_t SrcSlicePitch = Mul(SrcDim[1], SrcPitch);
+  size_t Width = Mul(Vol[2], ES);
+  size_t DstX = Mul(DstOff[2], ES);
+  size_t SrcX = Mul(SrcOff[2], ES);
+  for (size_t V : {Width, Vol[1], Vol[0], DstX, DstOff[1], DstOff[0], SrcX,
+                   SrcOff[1], SrcOff[0]})
+    InRange &= V <= Max;
+  if (!InRange)
+    return false;
+
+  DstLayout = {{static_cast<uint32_t>(DstX), static_cast<uint32_t>(DstOff[1]),
+                static_cast<uint32_t>(DstOff[0])},
+               DstPitch,
+               DstSlicePitch};
+  SrcLayout = {{static_cast<uint32_t>(SrcX), static_cast<uint32_t>(SrcOff[1]),
+                static_cast<uint32_t>(SrcOff[0])},
+               SrcPitch,
+               SrcSlicePitch};
+  Size = {static_cast<uint32_t>(Width), static_cast<uint32_t>(Vol[1]),
+          static_cast<uint32_t>(Vol[0])};
+  return true;
+}
+
+llvm::Error DeviceTy::dataMemcpyRect(
+    CopyDirTy Dir, void *Dst, const void *Src, const MemcpyRectTy &Rect,
+    DeviceTy *PeerDev, AsyncInfoTy *AsyncInfo,
+    llvm::function_ref<int32_t(void *, const void *, size_t, AsyncInfoTy &)>
+        CopyRow) {
+  // Fast path: one liboffload call.
+  ol_memcpy3d_layout_t DstLayout, SrcLayout;
+  ol_dimensions_t Size;
+  if (!Memcpy3DUnsupported && Rect.NumDims >= 2 && Rect.NumDims <= 3 &&
+      makeMemcpy3DArgs(Rect, DstLayout, SrcLayout, Size)) {
+    auto SupportedOrErr = dataMemcpy3D(Dir, Dst, DstLayout, Src, SrcLayout,
+                                       Size, PeerDev, AsyncInfo);
+    if (!SupportedOrErr)
+      return SupportedOrErr.takeError();
+    if (*SupportedOrErr)
+      return llvm::Error::success();
+  }
+
+  // Generic path: one copy per contiguous row.
+  // Without an AsyncInfo, use a local one and synchronize at the end.
+  std::optional<AsyncInfoTy> LocalAsyncInfo;
+  AsyncInfoTy *Info = AsyncInfo;
+  if (!Info) {
+    LocalAsyncInfo.emplace(*this);
+    Info = &*LocalAsyncInfo;
+  }
+
+  auto CopyCheckedRow = [&](void *DstAddr, const void *SrcAddr,
+                            size_t Length) -> llvm::Error {
+    if (CopyRow(DstAddr, SrcAddr, Length, *Info) != OFFLOAD_SUCCESS)
+      return createError(ErrorCode::BackendFailure,
+                         "failure in rectangular memcpy row copy: dst=%p "
+                         "src=%p size=%zu",
+                         DstAddr, SrcAddr, Length);
+    return llvm::Error::success();
+  };
+  if (llvm::Error Err = forEachMemcpyRectRow(Dst, Src, Rect, CopyCheckedRow))
+    return Err;
+
+  if (LocalAsyncInfo && LocalAsyncInfo->synchronize() != OFFLOAD_SUCCESS)
+    return createError(ErrorCode::BackendFailure,
+                       "failed to synchronize rectangular memcpy");
+  return llvm::Error::success();
+}
+
+llvm::Error DeviceTy::submitDataRect(void *TgtPtr, const void *HstPtr,
+                                     const MemcpyRectTy &Rect,
+                                     AsyncInfoTy *AsyncInfo) {
+  auto CopyRow = [&](void *Dst, const void *Src, size_t Length,
+                     AsyncInfoTy &Info) {
+    return submitData(Dst, const_cast<void *>(Src), Length, Info);
+  };
+  return dataMemcpyRect(CopyDirTy::HostToDevice, TgtPtr, HstPtr, Rect, nullptr,
+                        AsyncInfo, CopyRow);
+}
+
+llvm::Error DeviceTy::retrieveDataRect(void *HstPtr, const void *TgtPtr,
+                                       const MemcpyRectTy &Rect,
+                                       AsyncInfoTy *AsyncInfo) {
+  auto CopyRow = [&](void *Dst, const void *Src, size_t Length,
+                     AsyncInfoTy &Info) {
+    return retrieveData(Dst, const_cast<void *>(Src), Length, Info);
+  };
+  return dataMemcpyRect(CopyDirTy::DeviceToHost, HstPtr, TgtPtr, Rect, nullptr,
+                        AsyncInfo, CopyRow);
+}
+
+llvm::Error DeviceTy::dataExchangeRect(const void *SrcPtr, DeviceTy &DstDev,
+                                       void *DstPtr, const MemcpyRectTy &Rect,
+                                       AsyncInfoTy *AsyncInfo) {
+  auto CopyRow = [&](void *Dst, const void *Src, size_t Length,
+                     AsyncInfoTy &Info) {
+    return dataExchange(const_cast<void *>(Src), DstDev, Dst, Length, Info);
+  };
+  return dataMemcpyRect(CopyDirTy::DeviceToDevice, DstPtr, SrcPtr, Rect,
+                        &DstDev, AsyncInfo, CopyRow);
+}
+
 llvm::Expected<void *> DeviceTy::registerMemory(void *HstPtr, int64_t Size,
                                                 bool LockMemory) {
   void *LockedPtr = nullptr;
diff --git a/offload/libomptarget/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp
index 742f9bae927b3..f3848d294ab0f 100644
--- a/offload/libomptarget/OpenMP/API.cpp
+++ b/offload/libomptarget/OpenMP/API.cpp
@@ -42,7 +42,6 @@ using namespace llvm::omp::target::ompt;
 #endif
 using namespace llvm::omp::target::debug;
 
-
 void *targetAllocExplicit(size_t Size, int DeviceNum, int Kind,
                           const char *Name);
 void targetFreeExplicit(void *DevicePtr, int DeviceNum, int Kind,
@@ -545,6 +544,16 @@ EXTERN int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length,
   return Rc;
 }
 
+// Returns the device for \p DeviceNum, or nullptr if it is the host.
+static DeviceTy *getDeviceOrHost(int DeviceNum) {
+  if (isInitialDevice(DeviceNum))
+    return nullptr;
+  auto DevOrErr = PM->getDevice(DeviceNum);
+  if (!DevOrErr)
+    FATAL_MESSAGE(DeviceNum, "%s", toString(DevOrErr.takeError()).c_str());
+  return &*DevOrErr;
+}
+
 EXTERN int
 omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
                        int NumDims, const size_t *Volume,
@@ -572,36 +581,39 @@ omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
     return OFFLOAD_FAIL;
   }
 
-  int Rc;
-  if (NumDims == 1) {
-    Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0],
-                           ElementSize * DstOffsets[0],
-                           ElementSize * SrcOffsets[0], DstDevice, SrcDevice);
-  } else {
-    size_t DstSliceSize = ElementSize;
-    size_t SrcSliceSize = ElementSize;
-    for (int I = 1; I < NumDims; ++I) {
-      DstSliceSize *= DstDimensions[I];
-      SrcSliceSize *= SrcDimensions[I];
-    }
-
-    size_t DstOff = DstOffsets[0] * DstSliceSize;
-    size_t SrcOff = SrcOffsets[0] * SrcSliceSize;
-    for (size_t I = 0; I < Volume[0]; ++I) {
-      Rc = omp_target_memcpy_rect(
-          (char *)Dst + DstOff + DstSliceSize * I,
-          (char *)const_cast<void *>(Src) + SrcOff + SrcSliceSize * I,
-          ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1,
-          DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice);
-
-      if (Rc) {
-        ODBG(ODT_Interface)
-            << "Recursive call to " << __func__ << " returns unsuccessfully";
-        return Rc;
-      }
+  for (int I = 0; I < NumDims; ++I) {
+    if (!Volume[I]) {
+      ODBG(ODT_Interface) << "Call to " << __func__
+                          << " with zero volume, nothing to do";
+      return OFFLOAD_SUCCESS;
     }
   }
 
+  DeviceTy *DstDev = getDeviceOrHost(DstDevice);
+  DeviceTy *SrcDev = getDeviceOrHost(SrcDevice);
+
+  MemcpyRectTy Rect = {ElementSize, NumDims,       Volume,       DstOffsets,
+                       SrcOffsets,  DstDimensions, SrcDimensions};
+  llvm::Error Err = [&]() -> llvm::Error {
+    if (!DstDev && !SrcDev)
+      return forEachMemcpyRectRow(
+          Dst, Src, Rect,
+          [](void *DstAddr, const void *SrcAddr, size_t Length) {
+            memcpy(DstAddr, SrcAddr, Length);
+            return llvm::Error::success();
+          });
+    if (!SrcDev)
+      return DstDev->submitDataRect(Dst, Src, Rect, nullptr);
+    if (!DstDev)
+      return SrcDev->retrieveDataRect(Dst, Src, Rect, nullptr);
+    return SrcDev->dataExchangeRect(Src, *DstDev, Dst, Rect, nullptr);
+  }();
+
+  int Rc = OFFLOAD_SUCCESS;
+  if (Err) {
+    REPORT() << toString(std::move(Err));
+    Rc = OFFLOAD_FAIL;
+  }
   ODBG(ODT_Interface) << " returns " << Rc;
   return Rc;
 }
diff --git a/offload/test/api/omp_target_memcpy_rect_3d.c b/offload/test/api/omp_target_memcpy_rect_3d.c
new file mode 100644
index 0000000000000..e7a6a8c6a5413
--- /dev/null
+++ b/offload/test/api/omp_target_memcpy_rect_3d.c
@@ -0,0 +1,124 @@
+// RUN: %libomptarget-compile-and-run-generic | %fcheck-generic
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Reference implementation on host memory.
+static void ref_rect(char *dst, const char *src, size_t es, int nd,
+                     const size_t *vol, const size_t *doff, const size_t *soff,
+                     const size_t *ddim, const size_t *sdim) {
+  if (nd == 1) {
+    memcpy(dst + doff[0] * es, src + soff[0] * es, vol[0] * es);
+    return;
+  }
+  size_t dslice = es, sslice = es;
+  for (int i = 1; i < nd; ++i) {
+    dslice *= ddim[i];
+    sslice *= sdim[i];
+  }
+  for (size_t i = 0; i < vol[0]; ++i)
+    ref_rect(dst + (doff[0] + i) * dslice, src + (soff[0] + i) * sslice, es,
+             nd - 1, vol + 1, doff + 1, soff + 1, ddim + 1, sdim + 1);
+}
+
+enum Dir { H2H, H2D, D2H, D2D };
+
+static size_t total(int nd, const size_t *dim, size_t es) {
+  size_t n = es;
+  for (int i = 0; i < nd; ++i)
+    n *= dim[i];
+  return n;
+}
+
+// Run one copy in direction \p dir and compare against the reference.
+static int run_case(const char *name, enum Dir dir, int nd, size_t es,
+                    const size_t *vol, const size_t *doff, const size_t *soff,
+                    const size_t *ddim, const size_t *sdim) {
+  int host = omp_get_initial_device();
+  int dev = omp_get_default_device();
+  if (dev < 0 || dev >= omp_get_num_devices())
+    dev = host;
+
+  size_t sbytes = total(nd, sdim, es), dbytes = total(nd, ddim, es);
+  char *hsrc = malloc(sbytes), *href = calloc(1, dbytes),
+       *hout = calloc(1, dbytes);
+  for (size_t i = 0; i < sbytes; ++i)
+    hsrc[i] = (char)(i * 7 + 1);
+  ref_rect(href, hsrc, es, nd, vol, doff, soff, ddim, sdim);
+
+  int sdev = (dir == H2H || dir == H2D) ? host : dev;
+  int ddev = (dir == H2H || dir == D2H) ? host : dev;
+  char *src = sdev == host ? hsrc : omp_target_alloc(sbytes, sdev);
+  char *dst = ddev == host ? hout : omp_target_alloc(dbytes, ddev);
+  if (!src || !dst)
+    return 1;
+  if (sdev != host && omp_target_memcpy(src, hsrc, sbytes, 0, 0, sdev, host))
+    return 1;
+  if (ddev != host && omp_target_memcpy(dst, hout, dbytes, 0, 0, ddev, host))
+    return 1;
+
+  if (omp_target_memcpy_rect(dst, src, es, nd, vol, doff, soff, ddim, sdim,
+                             ddev, sdev)) {
+    printf("%s: rect copy failed\n", name);
+    return 1;
+  }
+  if (ddev != host && omp_target_memcpy(hout, dst, dbytes, 0, 0, host, ddev))
+    return 1;
+
+  int rc = memcmp(href, hout, dbytes) != 0;
+  if (rc)
+    printf("%s: mismatch\n", name);
+  if (sdev != host)
+    omp_target_free(src, sdev);
+  if (ddev != host)
+    omp_target_free(dst, ddev);
+  free(hsrc);
+  free(href);
+  free(hout);
+  return rc;
+}
+
+int main() {
+  // 1D
+  size_t v1[1] = {10}, o1[1] = {3}, s1[1] = {5}, d1[1] = {32};
+  // 2D: 3 rows of 5 elements
+  size_t v2[2] = {3, 5}, so2[2] = {1, 2}, do2[2] = {2, 1};
+  size_t sd2[2] = {6, 9}, dd2[2] = {7, 10};
+  // 3D
+  size_t v3[3] = {2, 3, 4}, so3[3] = {1, 2, 3}, do3[3] = {2, 1, 0};
+  size_t dim3[3] = {4, 6, 8};
+  // 4D (handled by the generic per-row path)
+  size_t v4[4] = {2, 2, 3, 4}, so4[4] = {1, 0, 1, 2}, do4[4] = {0, 1, 1, 1};
+  size_t dim4[4] = {3, 3, 4, 7};
+
+  static const char *dirs[] = {"H2H", "H2D", "D2H", "D2D"};
+  char name[64];
+  for (int d = H2H; d <= D2D; ++d) {
+    for (size_t es = 4; es <= 8; es += 4) {
+#define CASE(N, V, DO, SO, DD, SD)                                             \
+  snprintf(name, sizeof(name), "%dD %s es=%zu", N, dirs[d], es);               \
+  if (run_case(name, d, N, es, V, DO, SO, DD, SD))                             \
+    return 1;
+      CASE(1, v1, o1, s1, d1, d1)
+      CASE(2, v2, do2, so2, dd2, sd2)
+      CASE(3, v3, do3, so3, dim3, dim3)
+      CASE(4, v4, do4, so4, dim4, dim4)
+#undef CASE
+    }
+  }
+
+  // Nothing to copy: succeeds without looking at the devices.
+  size_t vz[2] = {0, 4}, oz[2] = {0, 0}, dz[2] = {4, 4};
+  char a, b;
+  if (omp_target_memcpy_rect(&a, &b, 1, 2, vz, oz, oz, dz, dz, 12345, 12345)) {
+    printf("zero volume failed\n");
+    return 1;
+  }
+
+  printf("PASS\n");
+  return 0;
+}
+
+// CHECK: PASS



More information about the llvm-branch-commits mailing list