[llvm] [offload] allow h2h olMemcpy with a queue (PR #212757)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 05:29:37 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Jan Trusiłło (311Volt)
<details>
<summary>Changes</summary>
olMemcpy currently disallows host-to-host copies with a queue, even though that is a desirable use case (e.g. to copy a buffer as a dependency for another operation)
This PR adds support for asynchronous host-to-host copies by adding `GenericDeviceTy::dataMemcpy` and the corresponding required `dataMemcpyImpl` plugin operation. Unlike `dataSubmit`, `dataRetrieve`, and `dataExchange`, this operation does not assign a device direction to either pointer:
- CUDA: Uses `cuMemcpyAsync`, which by itself the pointer types and preserves stream ordering. Dynamic CUDA symbol declarations and `_v2` lookup support are included.
- Level Zero: Uses the existing queue `memoryCopy` operation, which appends a memory-copy command to the command list.
- AMDGPU: No "enqueue a host-to-host copy" operation exists directly in that API, so we enqueue a `std::memcpy` callback through `pushHostCallback`.
- Host: Call `memcpy` directly.
The `olMemcpy` API documentation now states that queued host-to-host copies are ordered with other work in the queue. The public function signature is unchanged.
A new unit test executes an `h2d -> d2h -> h2h -> h2d -> d2h` operation chain, which should fail if the queue order isn't respected.
Assisted-by: Codex
---
Full diff: https://github.com/llvm/llvm-project/pull/212757.diff
12 Files Affected:
- (modified) offload/liboffload/API/Memory.td (+1-1)
- (modified) offload/liboffload/src/OffloadImpl.cpp (+3-5)
- (modified) offload/plugins-nextgen/amdgpu/src/rtl.cpp (+27)
- (modified) offload/plugins-nextgen/common/include/PluginInterface.h (+6)
- (modified) offload/plugins-nextgen/common/src/PluginInterface.cpp (+12)
- (modified) offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp (+2)
- (modified) offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h (+1)
- (modified) offload/plugins-nextgen/cuda/src/rtl.cpp (+14)
- (modified) offload/plugins-nextgen/host/src/rtl.cpp (+6)
- (modified) offload/plugins-nextgen/level_zero/include/L0Device.h (+2)
- (modified) offload/plugins-nextgen/level_zero/src/L0Device.cpp (+5)
- (modified) offload/unittests/OffloadAPI/memory/olMemcpy.cpp (+33)
``````````diff
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 3cd25a0f56161..71dd08c377dda 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -191,7 +191,7 @@ def olMemcpy : Function {
let desc = "Enqueue a memcpy operation.";
let details = [
"For host pointers, use the host device belonging to the OL_PLATFORM_BACKEND_HOST platform.",
- "If a queue is specified, at least one device must be a non-host device",
+ "If both pointers are on the host and a queue is specified, the memcpy is ordered with other work in the queue.",
"If a queue is not specified, the memcpy happens synchronously"
];
let params = [
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 63fe726fbe544..ba1b293537d4b 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -1078,12 +1078,10 @@ Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
if (!Queue) {
std::memcpy(DstPtr, SrcPtr, Size);
return Error::success();
- } else {
- return createOffloadError(
- ErrorCode::INVALID_ARGUMENT,
- "one of DstDevice and SrcDevice must be a non-host device if "
- "queue is specified");
}
+
+ return Queue->Device->Device->dataMemcpy(DstPtr, SrcPtr, Size,
+ Queue->AsyncInfo);
}
// If no queue is given the memcpy will be synchronous
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 65ef83c394f6a..7060d9d35a806 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2952,6 +2952,33 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
PinnedMemoryManager);
}
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+ struct MemcpyArgsTy {
+ void *DstPtr;
+ const void *SrcPtr;
+ int64_t Size;
+ };
+
+ AMDGPUStreamTy *Stream = nullptr;
+ if (auto Err = getStream(AsyncInfoWrapper, Stream))
+ return Err;
+
+ auto Args =
+ std::make_unique<MemcpyArgsTy>(MemcpyArgsTy{DstPtr, SrcPtr, Size});
+ if (auto Err = Stream->pushHostCallback(
+ [](void *Data) {
+ std::unique_ptr<MemcpyArgsTy> Args(
+ static_cast<MemcpyArgsTy *>(Data));
+ std::memcpy(Args->DstPtr, Args->SrcPtr, Args->Size);
+ },
+ Args.get()))
+ return Err;
+
+ Args.release();
+ return Plugin::success();
+ }
+
/// Exchange data between two devices within the plugin.
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
void *DstPtr, int64_t Size,
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 67ebfbc943fdc..afa979fa33eec 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1067,6 +1067,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
virtual Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+ /// Copy data between arbitrary memory locations.
+ Error dataMemcpy(void *DstPtr, const void *SrcPtr, int64_t Size,
+ __tgt_async_info *AsyncInfo);
+ virtual Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+
/// Instert a data fence between previous data operations and the following
/// operations if necessary for the device
virtual Error dataFence(__tgt_async_info *AsyncInfo) = 0;
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 7b821e77df179..e539cbfc55324 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1147,6 +1147,18 @@ Error GenericDeviceTy::dataRetrieve(void *HstPtr, const void *TgtPtr,
return Err;
}
+Error GenericDeviceTy::dataMemcpy(void *DstPtr, const void *SrcPtr,
+ int64_t Size, __tgt_async_info *AsyncInfo) {
+ if (Size == 0)
+ return Plugin::success();
+
+ AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
+
+ auto Err = dataMemcpyImpl(DstPtr, SrcPtr, Size, AsyncInfoWrapper);
+ AsyncInfoWrapper.finalize(Err);
+ return Err;
+}
+
Error GenericDeviceTy::dataExchange(const void *SrcPtr, GenericDeviceTy &DstDev,
void *DstPtr, int64_t Size,
__tgt_async_info *AsyncInfo) {
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
index 8436071a35dca..e3a854d5690c1 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
@@ -50,6 +50,7 @@ DLWRAP(cuMemAllocHost, 2)
DLWRAP(cuMemAllocManaged, 3)
DLWRAP(cuMemAllocAsync, 3)
+DLWRAP(cuMemcpyAsync, 4)
DLWRAP(cuMemcpyDtoDAsync, 4)
DLWRAP(cuMemcpyDtoH, 3)
DLWRAP(cuMemcpyDtoHAsync, 4)
@@ -137,6 +138,7 @@ static bool checkForCUDA() {
{"cuMemcpyHtoD", "cuMemcpyHtoD_v2"},
{"cuStreamDestroy", "cuStreamDestroy_v2"},
{"cuModuleGetGlobal", "cuModuleGetGlobal_v2"},
+ {"cuMemcpyAsync", "cuMemcpyAsync_v2"},
{"cuMemcpyDtoHAsync", "cuMemcpyDtoHAsync_v2"},
{"cuMemcpyDtoDAsync", "cuMemcpyDtoDAsync_v2"},
{"cuMemcpyHtoDAsync", "cuMemcpyHtoDAsync_v2"},
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
index 8597ea0055d78..a7524d417dded 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
@@ -458,6 +458,7 @@ CUresult cuMemAllocHost(void **, size_t);
CUresult cuMemAllocManaged(CUdeviceptr *, size_t, unsigned int);
CUresult cuMemAllocAsync(CUdeviceptr *, size_t, CUstream);
+CUresult cuMemcpyAsync(CUdeviceptr, CUdeviceptr, size_t, CUstream);
CUresult cuMemcpyDtoDAsync(CUdeviceptr, CUdeviceptr, size_t, CUstream);
CUresult cuMemcpyDtoH(void *, CUdeviceptr, size_t);
CUresult cuMemcpyDtoHAsync(void *, CUdeviceptr, size_t, CUstream);
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 9bf835b63813c..2db5557ec22d8 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -841,6 +841,20 @@ struct CUDADeviceTy : public GenericDeviceTy {
return Plugin::check(Res, "error in cuMemcpyDtoHAsync: %s");
}
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+ if (auto Err = setContext())
+ return Err;
+
+ CUstream Stream;
+ if (auto Err = getStream(AsyncInfoWrapper, Stream))
+ return Err;
+
+ CUresult Res =
+ cuMemcpyAsync((CUdeviceptr)DstPtr, (CUdeviceptr)SrcPtr, Size, Stream);
+ return Plugin::check(Res, "error in cuMemcpyAsync: %s");
+ }
+
/// Exchange data between two devices directly. We may use peer access if
/// the CUDA devices and driver allow them.
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 95ec3820f2657..d7ea85ae0881a 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -278,6 +278,12 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
return Plugin::success();
}
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+ std::memcpy(DstPtr, SrcPtr, Size);
+ return Plugin::success();
+ }
+
/// Exchange data between two devices within the plugin. This function is not
/// supported in this plugin.
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 3e8129d057095..3babe61795f02 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -542,6 +542,8 @@ class L0DeviceTy final : public GenericDeviceTy {
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override;
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstDev,
void *DstPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index 076dfa080f86e..8c769c1697984 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -378,6 +378,11 @@ Error L0DeviceTy::dataRetrieveImpl(void *HstPtr, const void *TgtPtr,
return Plugin::success();
}
+Error L0DeviceTy::dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) {
+ return enqueueMemCopy(DstPtr, SrcPtr, Size, AsyncInfoWrapper);
+}
+
Error L0DeviceTy::enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
AsyncInfoWrapperTy &AsyncInfoWrapper) {
__tgt_async_info *AsyncInfo = AsyncInfoWrapper;
diff --git a/offload/unittests/OffloadAPI/memory/olMemcpy.cpp b/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
index 3d210e3d6d015..38de00dd2267f 100644
--- a/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
@@ -98,6 +98,30 @@ TEST_P(olMemcpyTest, SuccessHtoHSync) {
}
}
+TEST_P(olMemcpyTest, SuccessHtoHQueuedOrdering) {
+ constexpr size_t Size = 1024;
+ void *Alloc;
+ std::vector<uint8_t> Input(Size, 42);
+ std::vector<uint8_t> Intermediate(Size, 0);
+ std::vector<uint8_t> Copied(Size, 0);
+ std::vector<uint8_t> Output(Size, 0);
+
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+ ASSERT_SUCCESS(olMemcpy(Queue, Alloc, Device, Input.data(), Host, Size));
+ ASSERT_SUCCESS(
+ olMemcpy(Queue, Intermediate.data(), Host, Alloc, Device, Size));
+ ASSERT_SUCCESS(
+ olMemcpy(Queue, Copied.data(), Host, Intermediate.data(), Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Alloc, Device, Copied.data(), Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Output.data(), Host, Alloc, Device, Size));
+ ASSERT_SUCCESS(olSyncQueue(Queue));
+
+ for (uint8_t Val : Output)
+ ASSERT_EQ(Val, 42);
+
+ ASSERT_SUCCESS(olMemFree(Alloc));
+}
+
TEST_P(olMemcpyTest, SuccessDtoHSync) {
constexpr size_t Size = 1024;
void *Alloc;
@@ -123,6 +147,15 @@ TEST_P(olMemcpyTest, SuccessSizeZero) {
ASSERT_SUCCESS(olMemcpy(nullptr, Output.data(), Host, Input.data(), Host, 0));
}
+TEST_P(olMemcpyTest, SuccessHtoHQueuedSizeZero) {
+ constexpr size_t Size = 1024;
+ std::vector<uint8_t> Input(Size, 42);
+ std::vector<uint8_t> Output(Size, 0);
+
+ ASSERT_SUCCESS(olMemcpy(Queue, Output.data(), Host, Input.data(), Host, 0));
+ ASSERT_SUCCESS(olSyncQueue(Queue));
+}
+
TEST_P(olMemcpyGlobalTest, SuccessRoundTrip) {
void *SourceMem;
ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
``````````
</details>
https://github.com/llvm/llvm-project/pull/212757
More information about the llvm-commits
mailing list