[llvm] [offload] allow h2h olMemcpy with a queue (PR #212757)
Jan Trusiłło via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 02:17:25 PDT 2026
https://github.com/311Volt updated https://github.com/llvm/llvm-project/pull/212757
>From 108a58fc3547d394d9b912515334272773b760c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Wed, 29 Jul 2026 11:38:25 +0000
Subject: [PATCH 1/2] [offload] allow h2h olMemcpy with a queue
---
offload/liboffload/API/Memory.td | 2 +-
offload/liboffload/src/OffloadImpl.cpp | 8 ++---
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 27 +++++++++++++++
.../common/include/PluginInterface.h | 6 ++++
.../common/src/PluginInterface.cpp | 12 +++++++
.../cuda/dynamic_cuda/cuda.cpp | 2 ++
.../plugins-nextgen/cuda/dynamic_cuda/cuda.h | 1 +
offload/plugins-nextgen/cuda/src/rtl.cpp | 14 ++++++++
offload/plugins-nextgen/host/src/rtl.cpp | 6 ++++
.../level_zero/include/L0Device.h | 2 ++
.../level_zero/src/L0Device.cpp | 5 +++
.../unittests/OffloadAPI/memory/olMemcpy.cpp | 33 +++++++++++++++++++
12 files changed, 112 insertions(+), 6 deletions(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 3cd25a0f56161..71dd08c377dda 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -191,7 +191,7 @@ def olMemcpy : Function {
let desc = "Enqueue a memcpy operation.";
let details = [
"For host pointers, use the host device belonging to the OL_PLATFORM_BACKEND_HOST platform.",
- "If a queue is specified, at least one device must be a non-host device",
+ "If both pointers are on the host and a queue is specified, the memcpy is ordered with other work in the queue.",
"If a queue is not specified, the memcpy happens synchronously"
];
let params = [
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 63fe726fbe544..ba1b293537d4b 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -1078,12 +1078,10 @@ Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
if (!Queue) {
std::memcpy(DstPtr, SrcPtr, Size);
return Error::success();
- } else {
- return createOffloadError(
- ErrorCode::INVALID_ARGUMENT,
- "one of DstDevice and SrcDevice must be a non-host device if "
- "queue is specified");
}
+
+ return Queue->Device->Device->dataMemcpy(DstPtr, SrcPtr, Size,
+ Queue->AsyncInfo);
}
// If no queue is given the memcpy will be synchronous
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 65ef83c394f6a..7060d9d35a806 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2952,6 +2952,33 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
PinnedMemoryManager);
}
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+ struct MemcpyArgsTy {
+ void *DstPtr;
+ const void *SrcPtr;
+ int64_t Size;
+ };
+
+ AMDGPUStreamTy *Stream = nullptr;
+ if (auto Err = getStream(AsyncInfoWrapper, Stream))
+ return Err;
+
+ auto Args =
+ std::make_unique<MemcpyArgsTy>(MemcpyArgsTy{DstPtr, SrcPtr, Size});
+ if (auto Err = Stream->pushHostCallback(
+ [](void *Data) {
+ std::unique_ptr<MemcpyArgsTy> Args(
+ static_cast<MemcpyArgsTy *>(Data));
+ std::memcpy(Args->DstPtr, Args->SrcPtr, Args->Size);
+ },
+ Args.get()))
+ return Err;
+
+ Args.release();
+ return Plugin::success();
+ }
+
/// Exchange data between two devices within the plugin.
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
void *DstPtr, int64_t Size,
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 67ebfbc943fdc..afa979fa33eec 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1067,6 +1067,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
virtual Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+ /// Copy data between arbitrary memory locations.
+ Error dataMemcpy(void *DstPtr, const void *SrcPtr, int64_t Size,
+ __tgt_async_info *AsyncInfo);
+ virtual Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+
/// Instert a data fence between previous data operations and the following
/// operations if necessary for the device
virtual Error dataFence(__tgt_async_info *AsyncInfo) = 0;
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 7b821e77df179..e539cbfc55324 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1147,6 +1147,18 @@ Error GenericDeviceTy::dataRetrieve(void *HstPtr, const void *TgtPtr,
return Err;
}
+Error GenericDeviceTy::dataMemcpy(void *DstPtr, const void *SrcPtr,
+ int64_t Size, __tgt_async_info *AsyncInfo) {
+ if (Size == 0)
+ return Plugin::success();
+
+ AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
+
+ auto Err = dataMemcpyImpl(DstPtr, SrcPtr, Size, AsyncInfoWrapper);
+ AsyncInfoWrapper.finalize(Err);
+ return Err;
+}
+
Error GenericDeviceTy::dataExchange(const void *SrcPtr, GenericDeviceTy &DstDev,
void *DstPtr, int64_t Size,
__tgt_async_info *AsyncInfo) {
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
index 8436071a35dca..e3a854d5690c1 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
@@ -50,6 +50,7 @@ DLWRAP(cuMemAllocHost, 2)
DLWRAP(cuMemAllocManaged, 3)
DLWRAP(cuMemAllocAsync, 3)
+DLWRAP(cuMemcpyAsync, 4)
DLWRAP(cuMemcpyDtoDAsync, 4)
DLWRAP(cuMemcpyDtoH, 3)
DLWRAP(cuMemcpyDtoHAsync, 4)
@@ -137,6 +138,7 @@ static bool checkForCUDA() {
{"cuMemcpyHtoD", "cuMemcpyHtoD_v2"},
{"cuStreamDestroy", "cuStreamDestroy_v2"},
{"cuModuleGetGlobal", "cuModuleGetGlobal_v2"},
+ {"cuMemcpyAsync", "cuMemcpyAsync_v2"},
{"cuMemcpyDtoHAsync", "cuMemcpyDtoHAsync_v2"},
{"cuMemcpyDtoDAsync", "cuMemcpyDtoDAsync_v2"},
{"cuMemcpyHtoDAsync", "cuMemcpyHtoDAsync_v2"},
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
index 8597ea0055d78..a7524d417dded 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
@@ -458,6 +458,7 @@ CUresult cuMemAllocHost(void **, size_t);
CUresult cuMemAllocManaged(CUdeviceptr *, size_t, unsigned int);
CUresult cuMemAllocAsync(CUdeviceptr *, size_t, CUstream);
+CUresult cuMemcpyAsync(CUdeviceptr, CUdeviceptr, size_t, CUstream);
CUresult cuMemcpyDtoDAsync(CUdeviceptr, CUdeviceptr, size_t, CUstream);
CUresult cuMemcpyDtoH(void *, CUdeviceptr, size_t);
CUresult cuMemcpyDtoHAsync(void *, CUdeviceptr, size_t, CUstream);
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 9bf835b63813c..2db5557ec22d8 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -841,6 +841,20 @@ struct CUDADeviceTy : public GenericDeviceTy {
return Plugin::check(Res, "error in cuMemcpyDtoHAsync: %s");
}
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+ if (auto Err = setContext())
+ return Err;
+
+ CUstream Stream;
+ if (auto Err = getStream(AsyncInfoWrapper, Stream))
+ return Err;
+
+ CUresult Res =
+ cuMemcpyAsync((CUdeviceptr)DstPtr, (CUdeviceptr)SrcPtr, Size, Stream);
+ return Plugin::check(Res, "error in cuMemcpyAsync: %s");
+ }
+
/// Exchange data between two devices directly. We may use peer access if
/// the CUDA devices and driver allow them.
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 95ec3820f2657..d7ea85ae0881a 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -278,6 +278,12 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
return Plugin::success();
}
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+ std::memcpy(DstPtr, SrcPtr, Size);
+ return Plugin::success();
+ }
+
/// Exchange data between two devices within the plugin. This function is not
/// supported in this plugin.
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 3e8129d057095..3babe61795f02 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -542,6 +542,8 @@ class L0DeviceTy final : public GenericDeviceTy {
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
+ Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) override;
Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstDev,
void *DstPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) override;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index 076dfa080f86e..8c769c1697984 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -378,6 +378,11 @@ Error L0DeviceTy::dataRetrieveImpl(void *HstPtr, const void *TgtPtr,
return Plugin::success();
}
+Error L0DeviceTy::dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
+ AsyncInfoWrapperTy &AsyncInfoWrapper) {
+ return enqueueMemCopy(DstPtr, SrcPtr, Size, AsyncInfoWrapper);
+}
+
Error L0DeviceTy::enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
AsyncInfoWrapperTy &AsyncInfoWrapper) {
__tgt_async_info *AsyncInfo = AsyncInfoWrapper;
diff --git a/offload/unittests/OffloadAPI/memory/olMemcpy.cpp b/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
index 3d210e3d6d015..38de00dd2267f 100644
--- a/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
@@ -98,6 +98,30 @@ TEST_P(olMemcpyTest, SuccessHtoHSync) {
}
}
+TEST_P(olMemcpyTest, SuccessHtoHQueuedOrdering) {
+ constexpr size_t Size = 1024;
+ void *Alloc;
+ std::vector<uint8_t> Input(Size, 42);
+ std::vector<uint8_t> Intermediate(Size, 0);
+ std::vector<uint8_t> Copied(Size, 0);
+ std::vector<uint8_t> Output(Size, 0);
+
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+ ASSERT_SUCCESS(olMemcpy(Queue, Alloc, Device, Input.data(), Host, Size));
+ ASSERT_SUCCESS(
+ olMemcpy(Queue, Intermediate.data(), Host, Alloc, Device, Size));
+ ASSERT_SUCCESS(
+ olMemcpy(Queue, Copied.data(), Host, Intermediate.data(), Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Alloc, Device, Copied.data(), Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Output.data(), Host, Alloc, Device, Size));
+ ASSERT_SUCCESS(olSyncQueue(Queue));
+
+ for (uint8_t Val : Output)
+ ASSERT_EQ(Val, 42);
+
+ ASSERT_SUCCESS(olMemFree(Alloc));
+}
+
TEST_P(olMemcpyTest, SuccessDtoHSync) {
constexpr size_t Size = 1024;
void *Alloc;
@@ -123,6 +147,15 @@ TEST_P(olMemcpyTest, SuccessSizeZero) {
ASSERT_SUCCESS(olMemcpy(nullptr, Output.data(), Host, Input.data(), Host, 0));
}
+TEST_P(olMemcpyTest, SuccessHtoHQueuedSizeZero) {
+ constexpr size_t Size = 1024;
+ std::vector<uint8_t> Input(Size, 42);
+ std::vector<uint8_t> Output(Size, 0);
+
+ ASSERT_SUCCESS(olMemcpy(Queue, Output.data(), Host, Input.data(), Host, 0));
+ ASSERT_SUCCESS(olSyncQueue(Queue));
+}
+
TEST_P(olMemcpyGlobalTest, SuccessRoundTrip) {
void *SourceMem;
ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
>From 19fe3f6b6e3652780ed619c1b91412986eadc52d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Mon, 3 Aug 2026 09:16:59 +0000
Subject: [PATCH 2/2] add olMemAllocHost variant of SuccessHtoHQueuedOrdering
---
.../unittests/OffloadAPI/memory/olMemcpy.cpp | 38 +++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/offload/unittests/OffloadAPI/memory/olMemcpy.cpp b/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
index 38de00dd2267f..b4f98aa331f8b 100644
--- a/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemcpy.cpp
@@ -10,6 +10,8 @@
#include <OffloadAPI.h>
#include <gtest/gtest.h>
+#include <cstring>
+
using olMemcpyTest = OffloadQueueTest;
OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olMemcpyTest);
@@ -122,6 +124,42 @@ TEST_P(olMemcpyTest, SuccessHtoHQueuedOrdering) {
ASSERT_SUCCESS(olMemFree(Alloc));
}
+TEST_P(olMemcpyTest, SuccessHtoHQueuedOrderingHostAlloc) {
+ constexpr size_t Size = 1024;
+ void *Alloc;
+ void *Input;
+ void *Intermediate;
+ void *Copied;
+ void *Output;
+
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+ ASSERT_SUCCESS(olMemAllocHost(Device, Size, &Input));
+ ASSERT_SUCCESS(olMemAllocHost(Device, Size, &Intermediate));
+ ASSERT_SUCCESS(olMemAllocHost(Device, Size, &Copied));
+ ASSERT_SUCCESS(olMemAllocHost(Device, Size, &Output));
+
+ std::memset(Input, 42, Size);
+ std::memset(Intermediate, 0, Size);
+ std::memset(Copied, 0, Size);
+ std::memset(Output, 0, Size);
+
+ ASSERT_SUCCESS(olMemcpy(Queue, Alloc, Device, Input, Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Intermediate, Host, Alloc, Device, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Copied, Host, Intermediate, Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Alloc, Device, Copied, Host, Size));
+ ASSERT_SUCCESS(olMemcpy(Queue, Output, Host, Alloc, Device, Size));
+ ASSERT_SUCCESS(olSyncQueue(Queue));
+
+ for (size_t I = 0; I < Size; ++I)
+ ASSERT_EQ(static_cast<uint8_t *>(Output)[I], 42);
+
+ ASSERT_SUCCESS(olMemFree(Output));
+ ASSERT_SUCCESS(olMemFree(Copied));
+ ASSERT_SUCCESS(olMemFree(Intermediate));
+ ASSERT_SUCCESS(olMemFree(Input));
+ ASSERT_SUCCESS(olMemFree(Alloc));
+}
+
TEST_P(olMemcpyTest, SuccessDtoHSync) {
constexpr size_t Size = 1024;
void *Alloc;
More information about the llvm-commits
mailing list