[llvm] [offload][l0] Synchronize seed upload in `olMemFill` L0 slow path (PR #215270)
Jan Trusiłło via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 05:07:49 PDT 2026
https://github.com/311Volt updated https://github.com/llvm/llvm-project/pull/215270
>From 2bf3e8fcd0eca47550b7c81995f5d8dbefb302d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 6 Aug 2026 13:39:18 +0000
Subject: [PATCH 1/6] [offload][l0] Stage non-native memory fill seeds per
queue
---
offload/liboffload/API/Memory.td | 6 +-
.../level_zero/include/L0Context.h | 22 ---
.../level_zero/include/L0Defs.h | 2 +
.../level_zero/include/L0Device.h | 2 -
.../level_zero/include/L0Memory.h | 12 --
.../level_zero/include/L0Plugin.h | 7 -
.../level_zero/include/L0Queue.h | 29 ++--
.../level_zero/src/L0Context.cpp | 10 --
.../level_zero/src/L0Device.cpp | 2 -
.../level_zero/src/L0Plugin.cpp | 2 -
.../level_zero/src/L0Queue.cpp | 128 ++++++++++--------
.../unittests/OffloadAPI/memory/olMemFill.cpp | 9 ++
12 files changed, 109 insertions(+), 122 deletions(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 71dd08c377dda..a8e8456760f0a 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -211,6 +211,7 @@ def olMemFill : Function {
"Filling with patterns of a different size than 1, 2 or 4 bytes may be less performant",
"The destination pointer and queue must be associated with the same device",
"The fill size must be a multiple of the pattern size",
+ "The pattern size must not exceed 1024 bytes",
];
let params = [
Param<"ol_queue_handle_t", "Queue", "handle of the queue", PARAM_IN_OPTIONAL>,
@@ -220,7 +221,10 @@ def olMemFill : Function {
Param<"size_t", "FillSize", "number of bytes to fill", PARAM_IN>,
];
let returns = [
- Return<"OL_ERRC_INVALID_SIZE", ["`FillSize % PatternSize != 0`"]>
+ Return<"OL_ERRC_INVALID_SIZE", [
+ "`FillSize % PatternSize != 0`",
+ "`PatternSize > 1024`",
+ ]>
];
}
diff --git a/offload/plugins-nextgen/level_zero/include/L0Context.h b/offload/plugins-nextgen/level_zero/include/L0Context.h
index 7bd082571b1a1..f33f81e9df68d 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Context.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Context.h
@@ -15,31 +15,11 @@
#include "L0Event.h"
#include "L0Memory.h"
-#include "PerThreadTable.h"
namespace llvm::omp::target::plugin {
class LevelZeroPluginTy;
-class L0ContextTLSTy {
- StagingBufferTy StagingBuffer;
-
-public:
- StagingBufferTy &getStagingBuffer() { return StagingBuffer; }
- const StagingBufferTy &getStagingBuffer() const { return StagingBuffer; }
-
- Error deinit() { return StagingBuffer.clear(); }
-};
-
-struct L0ContextTLSTableTy
- : public PerThreadContainer<
- std::unordered_map<ze_context_handle_t, L0ContextTLSTy>> {
- Error deinit() {
- return PerThreadTable::deinit(
- [](L0ContextTLSTy &Entry) -> auto { return Entry.deinit(); });
- }
-};
-
/// Driver and context-specific resources. We assume a single context per
/// driver.
class L0ContextTy {
@@ -89,8 +69,6 @@ class L0ContextTy {
LevelZeroPluginTy &getPlugin() const { return Plugin; }
- StagingBufferTy &getStagingBuffer();
-
/// Add imported external pointer region.
void addImported(void *Ptr, size_t Size) {
(void)ImportedPtrs.try_emplace(reinterpret_cast<uintptr_t>(Ptr), Size);
diff --git a/offload/plugins-nextgen/level_zero/include/L0Defs.h b/offload/plugins-nextgen/level_zero/include/L0Defs.h
index 1c616b93b779f..ddc91b67860cf 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Defs.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Defs.h
@@ -31,6 +31,8 @@ constexpr size_t L0DefaultAlignment = 0;
constexpr size_t L0StagingBufferSize = (1 << 14);
/// Default staging buffer count.
constexpr size_t L0StagingBufferCount = 64;
+/// Maximum memory fill pattern size supported by the Offload API.
+constexpr size_t L0MaxMemFillPatternSize = 1024;
/// USM allocation threshold where preallocation does not pay off (128MB).
constexpr size_t L0UsmPreAllocThreshold = (128 << 20);
/// Host USM allocation threshold where preallocation does not pay off (8MB).
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 9936cfd77fbbf..95ca54b062053 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -463,8 +463,6 @@ class L0DeviceTy final : public GenericDeviceTy {
return L0Context.getEventPool().releaseEventObject(EventObj);
}
- StagingBufferTy &getStagingBuffer() { return L0Context.getStagingBuffer(); }
-
bool supportsLargeMem() const { return L0Context.supportsLargeMem(); }
/// Returns the Queue from an async info object, or creates a new one if
diff --git a/offload/plugins-nextgen/level_zero/include/L0Memory.h b/offload/plugins-nextgen/level_zero/include/L0Memory.h
index 8f2f3422b770e..4f0ed23ba1215 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Memory.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Memory.h
@@ -463,8 +463,6 @@ class StagingBufferTy {
return Plugin::success();
}
- bool initialized() const { return Context != nullptr; }
-
void init(ze_context_handle_t ContextIn, size_t SizeIn, size_t CountIn) {
Context = ContextIn;
Size = SizeIn;
@@ -473,13 +471,6 @@ class StagingBufferTy {
void reset() { Offset = 0; }
- /// Always return the first buffer.
- Expected<void *> get() {
- if (Size == 0 || Count == 0)
- return nullptr;
- return Buffers.empty() ? addBuffers() : Buffers.front();
- }
-
/// Return the next available buffer.
Expected<void *> getNext() {
void *Ret = nullptr;
@@ -503,9 +494,6 @@ class StagingBufferTy {
Offset += Size;
return Ret;
}
-
- /// Return either a fixed buffer or next buffer.
- Expected<void *> get(bool Next) { return Next ? getNext() : get(); }
};
} // namespace llvm::omp::target::plugin
diff --git a/offload/plugins-nextgen/level_zero/include/L0Plugin.h b/offload/plugins-nextgen/level_zero/include/L0Plugin.h
index 56a708b7609a4..e48e689b44282 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Plugin.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Plugin.h
@@ -55,9 +55,6 @@ class LevelZeroPluginTy final : public GenericPluginTy {
/// Context (and Driver) specific data.
std::list<L0ContextTy> ContextList;
- // Table containing per-thread information for each Context using TLS.
- L0ContextTLSTableTy ContextTLSTable;
-
/// L0 plugin options.
L0OptionsTy Options;
@@ -69,10 +66,6 @@ class LevelZeroPluginTy final : public GenericPluginTy {
LevelZeroPluginTy() : GenericPluginTy(getTripleArch()) {}
virtual ~LevelZeroPluginTy() = default;
- L0ContextTLSTy &getContextTLS(ze_context_handle_t Context) {
- return ContextTLSTable.get(Context);
- }
-
const L0OptionsTy &getOptions() { return Options; }
const L0DeviceTy &getDeviceFromId(int32_t DeviceId) const {
diff --git a/offload/plugins-nextgen/level_zero/include/L0Queue.h b/offload/plugins-nextgen/level_zero/include/L0Queue.h
index 4ef2232c8da06..a1af2578a06a7 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Queue.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Queue.h
@@ -20,6 +20,7 @@
#include <tuple>
#include "L0CmdListManager.h"
+#include "L0Memory.h"
#include "L0Options.h"
namespace llvm::omp::target::plugin {
@@ -36,6 +37,8 @@ class L0QueueTy {
L0CmdListManagerTy *CmdList = nullptr;
/// Whether the queue is in-order or out-of-order.
bool CreateQueueInOrder;
+ /// Host buffers that remain valid until this queue completes.
+ StagingBufferTy StagingBuffer;
public:
L0QueueTy(L0DeviceTy &Device, bool IsInorder = true)
@@ -47,8 +50,8 @@ class L0QueueTy {
Error init();
Error deinit();
- Error synchronize() { return synchronizeImpl(); }
- Expected<bool> hasPendingWork() { return hasPendingWorkImpl(); }
+ Error synchronize();
+ Expected<bool> hasPendingWork();
Error memoryCopy(void *Dst, const void *Src, size_t Size) {
if (Size == 0)
@@ -67,9 +70,8 @@ class L0QueueTy {
return dataSubmitImpl(TgtPtr, HstPtr, Size);
}
- // Enqueue a memory fill command. Supports arbitrary pattern sizes, including
- // non-power-of-two sizes, by falling back to a less performant software fill
- // if necessary.
+ // Enqueue a memory fill command. Unsupported native patterns are replicated
+ // using ordered memory copies.
Error memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
size_t Size);
@@ -141,6 +143,8 @@ class L0QueueTy {
size_t PatternSize, size_t Size) {
return CmdList->appendMemoryFill(Ptr, Pattern, PatternSize, Size);
}
+ virtual Error memoryFillFallbackImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size);
virtual Error memoryPrefetchImpl(const void *Ptr, size_t Size) {
return CmdList->appendMemoryPrefetch(Ptr, Size);
}
@@ -154,12 +158,8 @@ class L0QueueTy {
}
private:
- /// Fallback fill for host-accessible target memory: replicate the pattern
- /// directly on the host with std::copy_n.
- Error memoryFillHostImpl(void *Ptr, const void *Pattern, size_t PatternSize,
- size_t Size);
- /// Fallback fill for non-host-accessible target memory: seed the pattern
- /// once and grow the filled region via device copies, doubling each time.
+ /// Fallback fill that seeds the pattern once and grows the filled region via
+ /// device copies, doubling each time.
Error memoryFillReplicateImpl(void *Ptr, const void *Pattern,
size_t PatternSize, size_t Size);
};
@@ -267,6 +267,13 @@ class L0SyncQueueTy : public L0InorderQueueTy {
Error launchKernelImpl(ze_kernel_handle_t Kernel,
L0LaunchEnvTy &KEnv) override;
Error hostCallImpl(void (*Callback)(void *), void *UserData) override;
+ Error memoryFillFallbackImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) override;
+
+private:
+ /// Fill host-accessible memory directly from the calling thread.
+ Error memoryFillHostImpl(void *Ptr, const void *Pattern, size_t PatternSize,
+ size_t Size);
};
/// Simple cache for queue objects.
diff --git a/offload/plugins-nextgen/level_zero/src/L0Context.cpp b/offload/plugins-nextgen/level_zero/src/L0Context.cpp
index be21a2fdc22ed..7809b4b24e192 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Context.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Context.cpp
@@ -81,14 +81,4 @@ Error L0ContextTy::deinit() {
return Plugin::success();
}
-StagingBufferTy &L0ContextTy::getStagingBuffer() {
- auto &TLS = Plugin.getContextTLS(getZeContext());
- auto &Buffer = TLS.getStagingBuffer();
- const auto &Options = Plugin.getOptions();
- if (!Buffer.initialized())
- Buffer.init(getZeContext(), Options.StagingBufferSize,
- Options.StagingBufferCount);
- return Buffer;
-}
-
} // namespace llvm::omp::target::plugin
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index c6cc578adb656..4b7c4bc7ce14e 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -282,7 +282,6 @@ Error L0DeviceTy::synchronizeImpl(__tgt_async_info &AsyncInfo,
if (ReleaseQueue) {
releaseQueue(Queue);
- getStagingBuffer().reset();
AsyncInfo.Queue = nullptr;
}
@@ -317,7 +316,6 @@ Error L0DeviceTy::queryAsyncImpl(__tgt_async_info &AsyncInfo, bool ReleaseQueue,
if (ReleaseQueue) {
releaseQueue(Queue);
- getStagingBuffer().reset();
AsyncInfo.Queue = nullptr;
}
diff --git a/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp b/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
index 5723644c6763c..d3c98579af33d 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
@@ -133,8 +133,6 @@ Expected<int32_t> LevelZeroPluginTy::initImpl() {
Error LevelZeroPluginTy::deinitImpl() {
ODBG(OLDT_Deinit) << "Deinit Level0 plugin!";
- if (auto Err = ContextTLSTable.deinit())
- return Err;
for (auto &Context : ContextList)
if (auto Err = Context.deinit())
return Err;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index a27024ad66d19..7e24d8eeae615 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -18,7 +18,6 @@
#include "llvm/Support/MathExtras.h"
#include <algorithm>
-#include <vector>
namespace llvm::omp::target::plugin {
@@ -29,22 +28,46 @@ Error L0QueueTy::init() {
if (!CmdListOrErr)
return CmdListOrErr.takeError();
CmdList = *CmdListOrErr;
+
+ const auto &Options = Device.getPlugin().getOptions();
+ StagingBuffer.init(
+ Device.getZeContext(),
+ std::max(Options.StagingBufferSize, L0MaxMemFillPatternSize),
+ Options.StagingBufferCount);
return initImpl();
}
Error L0QueueTy::deinit() {
- if (auto Err = deinitImpl())
- return Err;
+ Error AllErrors = deinitImpl();
reset();
+ if (auto Err = StagingBuffer.clear())
+ AllErrors = joinErrors(std::move(AllErrors), std::move(Err));
+
if (CmdList)
if (auto Err = Device.releaseCmdListManager(CmdList))
- return Err;
+ AllErrors = joinErrors(std::move(AllErrors), std::move(Err));
CmdList = nullptr;
+ return AllErrors;
+}
+
+Error L0QueueTy::synchronize() {
+ if (auto Err = synchronizeImpl())
+ return Err;
+ StagingBuffer.reset();
return Plugin::success();
}
+Expected<bool> L0QueueTy::hasPendingWork() {
+ auto PendingWorkOrErr = hasPendingWorkImpl();
+ if (!PendingWorkOrErr)
+ return PendingWorkOrErr.takeError();
+ if (!*PendingWorkOrErr)
+ StagingBuffer.reset();
+ return *PendingWorkOrErr;
+}
+
Error L0QueueTy::dispatchLaunchKernel(ze_kernel_handle_t Kernel,
L0LaunchEnvTy &KEnv,
ze_event_handle_t SignalEvent,
@@ -64,6 +87,11 @@ Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
if (Size == 0 || PatternSize == 0)
return Plugin::success();
+ if (PatternSize > L0MaxMemFillPatternSize)
+ return Plugin::error(
+ ErrorCode::INVALID_SIZE,
+ "memory fill pattern size exceeds the 1024-byte maximum");
+
if (llvm::isPowerOf2_64(PatternSize) && (Size % PatternSize == 0) &&
PatternSize <= Device.getMaxMemFillPatternSize()) {
// Native L0 memory fill is possible directly.
@@ -82,65 +110,41 @@ Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
// detection of repeating power-of-two patterns could be added here to allow
// native L0 memory fill for those cases as well.
- // Native L0 fill cannot handle this pattern size, but target memory is
- // host-accessible, so fall back to a software fill.
- const auto TgtType = Device.getMemAllocType(Ptr);
- if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
- return memoryFillHostImpl(Ptr, Pattern, PatternSize, Size);
+ return memoryFillFallbackImpl(Ptr, Pattern, PatternSize, Size);
+}
- // We know at this point that TgtType == ZE_MEMORY_TYPE_DEVICE.
- // Native fill and software fill are both impossible.
- // Seed the pattern once and grow the filled region with device copies,
- // doubling the amount copied each time.
+Error L0QueueTy::memoryFillFallbackImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
return memoryFillReplicateImpl(Ptr, Pattern, PatternSize, Size);
}
-Error L0QueueTy::memoryFillHostImpl(void *Ptr, const void *Pattern,
- size_t PatternSize, size_t Size) {
- auto *Dst = static_cast<unsigned char *>(Ptr);
- const auto *Pat = static_cast<const unsigned char *>(Pattern);
- // Seed the pattern once.
- std::copy_n(Pat, PatternSize, Dst);
- // Replicate the pattern until it fills the entire destination.
- for (size_t Offset = PatternSize; Offset < Size; ++Offset) {
+static void extendPattern(unsigned char *Dst, size_t Size, const void *Pattern,
+ size_t PatternSize) {
+ assert(Size >= PatternSize && Size % PatternSize == 0 &&
+ "Invalid pattern extension size");
+ std::copy_n(static_cast<const unsigned char *>(Pattern), PatternSize, Dst);
+ for (size_t Offset = PatternSize; Offset < Size; ++Offset)
Dst[Offset] = Dst[Offset - PatternSize];
- }
- return Plugin::success();
-}
-
-/// Replicate the pattern in \p Buf (of \p Size bytes) on the host until it is
-/// at least \p MinExtendedSize bytes long. The result is
-/// never larger than max(Size, 2 * MinExtendedSize).
-static std::vector<unsigned char> extendPattern(unsigned char *Buf, size_t Size,
- size_t MinExtendedSize) {
- assert(Size > 0 && MinExtendedSize > 0 &&
- "Invalid pattern size or extension size");
- const size_t NumPatterns =
- std::max(static_cast<size_t>(1), (MinExtendedSize + Size - 1) / Size);
- std::vector<unsigned char> Extended(NumPatterns * Size);
- // Seed the pattern.
- std::copy_n(Buf, Size, Extended.begin());
- // Replicate the pattern until we reach the desired size.
- for (size_t Offset = Size; Offset < Extended.size(); ++Offset) {
- Extended[Offset] = Extended[Offset - Size];
- }
- return Extended;
}
Error L0QueueTy::memoryFillReplicateImpl(void *Ptr, const void *Pattern,
size_t PatternSize, size_t Size) {
auto *Dst = static_cast<unsigned char *>(Ptr);
- // Grow the pattern on the host first - avoids several inefficient small
- // device copies.
- constexpr size_t MinExtendedSeedSize = 1024;
- const auto ExtendedPattern =
- extendPattern(static_cast<unsigned char *>(const_cast<void *>(Pattern)),
- PatternSize, std::min(Size, MinExtendedSeedSize));
+ auto SeedOrErr = StagingBuffer.getNext();
+ if (!SeedOrErr)
+ return SeedOrErr.takeError();
+ auto *Seed = static_cast<unsigned char *>(*SeedOrErr);
+ if (!Seed)
+ return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+ "failed to allocate a memory fill staging buffer");
- // Seed the (extended) pattern once using dataSubmit.
- size_t BytesFilled = std::min(ExtendedPattern.size(), Size);
- if (auto Err = dataSubmit(Dst, ExtendedPattern.data(), BytesFilled))
+ // Keep the seed pattern-aligned so each subsequent copy preserves it.
+ const size_t SeedLimit = std::min(Size, L0MaxMemFillPatternSize);
+ size_t BytesFilled = (SeedLimit / PatternSize) * PatternSize;
+ extendPattern(Seed, BytesFilled, Pattern, PatternSize);
+
+ if (auto Err = memoryCopy(Dst, Seed, BytesFilled))
return Err;
// Clone the seed, doubling each time, until it fills the entire destination.
@@ -276,7 +280,7 @@ Error L0AsyncQueueTy::dataRetrieveImpl(void *HstPtr, const void *TgtPtr,
static_cast<size_t>(Size) <=
Device.getPlugin().getOptions().StagingBufferSize &&
Device.getMemAllocType(HstPtr) != ZE_MEMORY_TYPE_HOST) {
- auto PtrOrErr = Device.getStagingBuffer().get(/*IsAsync*/ true);
+ auto PtrOrErr = StagingBuffer.getNext();
if (!PtrOrErr)
return PtrOrErr.takeError();
DstPtr = *PtrOrErr;
@@ -307,7 +311,7 @@ Error L0AsyncQueueTy::dataSubmitImpl(void *TgtPtr, const void *HstPtr,
static_cast<size_t>(Size) <=
Device.getPlugin().getOptions().StagingBufferSize &&
Device.getMemAllocType(HstPtr) != ZE_MEMORY_TYPE_HOST) {
- auto PtrOrErr = Device.getStagingBuffer().get(/*IsAsync*/ true);
+ auto PtrOrErr = StagingBuffer.getNext();
if (!PtrOrErr)
return PtrOrErr.takeError();
SrcPtr = *PtrOrErr;
@@ -457,6 +461,24 @@ Error L0SyncQueueTy::hostCallImpl(void (*Callback)(void *), void *UserData) {
return CmdList->hostSynchronize();
}
+Error L0SyncQueueTy::memoryFillFallbackImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ const auto TgtType = Device.getMemAllocType(Ptr);
+ if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
+ return memoryFillHostImpl(Ptr, Pattern, PatternSize, Size);
+ return L0QueueTy::memoryFillFallbackImpl(Ptr, Pattern, PatternSize, Size);
+}
+
+Error L0SyncQueueTy::memoryFillHostImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ auto *Dst = static_cast<unsigned char *>(Ptr);
+ const auto *Pat = static_cast<const unsigned char *>(Pattern);
+ std::copy_n(Pat, PatternSize, Dst);
+ for (size_t Offset = PatternSize; Offset < Size; ++Offset)
+ Dst[Offset] = Dst[Offset - PatternSize];
+ return Plugin::success();
+}
+
// L0QueueCache implementation.
Expected<L0QueueTy *> L0QueueCacheTy::getQueue() {
{
diff --git a/offload/unittests/OffloadAPI/memory/olMemFill.cpp b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
index 3437b78a9548c..3ef3e0072e3a0 100644
--- a/offload/unittests/OffloadAPI/memory/olMemFill.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
@@ -210,6 +210,15 @@ TEST_P(olMemFillTest, InvalidPatternSizeLargerThanFillSize) {
olMemFree(Alloc);
}
+TEST(olMemFillValidationTest, InvalidPatternSizeTooLarge) {
+ constexpr size_t Size = 1025;
+ std::array<unsigned char, Size> Alloc{};
+ std::array<unsigned char, Size> Pattern{};
+ ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
+ olMemFill(nullptr, Alloc.data(), Pattern.size(), Pattern.data(),
+ Alloc.size()));
+}
+
// Even though L0, CUDA and HSA do not support non-power-of-two patterns,
// plugins are currently expected to handle arbitrary pattern sizes.
// The following tests are intended to cover the fallback paths
>From b242de3a2c72d97b3f32db7a69cf1225cba75284 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Mon, 10 Aug 2026 15:30:24 +0000
Subject: [PATCH 2/6] restore old StagingBuffer, sync on seed upload
---
offload/liboffload/API/Memory.td | 6 +-
.../level_zero/include/L0Context.h | 22 ++++++
.../level_zero/include/L0Defs.h | 2 -
.../level_zero/include/L0Device.h | 2 +
.../level_zero/include/L0Memory.h | 12 ++++
.../level_zero/include/L0Plugin.h | 7 ++
.../level_zero/include/L0Queue.h | 7 +-
.../level_zero/src/L0Context.cpp | 10 +++
.../level_zero/src/L0Device.cpp | 2 +
.../level_zero/src/L0Plugin.cpp | 2 +
.../level_zero/src/L0Queue.cpp | 68 ++++++-------------
.../unittests/OffloadAPI/memory/olMemFill.cpp | 9 ---
12 files changed, 82 insertions(+), 67 deletions(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index a8e8456760f0a..71dd08c377dda 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -211,7 +211,6 @@ def olMemFill : Function {
"Filling with patterns of a different size than 1, 2 or 4 bytes may be less performant",
"The destination pointer and queue must be associated with the same device",
"The fill size must be a multiple of the pattern size",
- "The pattern size must not exceed 1024 bytes",
];
let params = [
Param<"ol_queue_handle_t", "Queue", "handle of the queue", PARAM_IN_OPTIONAL>,
@@ -221,10 +220,7 @@ def olMemFill : Function {
Param<"size_t", "FillSize", "number of bytes to fill", PARAM_IN>,
];
let returns = [
- Return<"OL_ERRC_INVALID_SIZE", [
- "`FillSize % PatternSize != 0`",
- "`PatternSize > 1024`",
- ]>
+ Return<"OL_ERRC_INVALID_SIZE", ["`FillSize % PatternSize != 0`"]>
];
}
diff --git a/offload/plugins-nextgen/level_zero/include/L0Context.h b/offload/plugins-nextgen/level_zero/include/L0Context.h
index f33f81e9df68d..7bd082571b1a1 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Context.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Context.h
@@ -15,11 +15,31 @@
#include "L0Event.h"
#include "L0Memory.h"
+#include "PerThreadTable.h"
namespace llvm::omp::target::plugin {
class LevelZeroPluginTy;
+class L0ContextTLSTy {
+ StagingBufferTy StagingBuffer;
+
+public:
+ StagingBufferTy &getStagingBuffer() { return StagingBuffer; }
+ const StagingBufferTy &getStagingBuffer() const { return StagingBuffer; }
+
+ Error deinit() { return StagingBuffer.clear(); }
+};
+
+struct L0ContextTLSTableTy
+ : public PerThreadContainer<
+ std::unordered_map<ze_context_handle_t, L0ContextTLSTy>> {
+ Error deinit() {
+ return PerThreadTable::deinit(
+ [](L0ContextTLSTy &Entry) -> auto { return Entry.deinit(); });
+ }
+};
+
/// Driver and context-specific resources. We assume a single context per
/// driver.
class L0ContextTy {
@@ -69,6 +89,8 @@ class L0ContextTy {
LevelZeroPluginTy &getPlugin() const { return Plugin; }
+ StagingBufferTy &getStagingBuffer();
+
/// Add imported external pointer region.
void addImported(void *Ptr, size_t Size) {
(void)ImportedPtrs.try_emplace(reinterpret_cast<uintptr_t>(Ptr), Size);
diff --git a/offload/plugins-nextgen/level_zero/include/L0Defs.h b/offload/plugins-nextgen/level_zero/include/L0Defs.h
index ddc91b67860cf..1c616b93b779f 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Defs.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Defs.h
@@ -31,8 +31,6 @@ constexpr size_t L0DefaultAlignment = 0;
constexpr size_t L0StagingBufferSize = (1 << 14);
/// Default staging buffer count.
constexpr size_t L0StagingBufferCount = 64;
-/// Maximum memory fill pattern size supported by the Offload API.
-constexpr size_t L0MaxMemFillPatternSize = 1024;
/// USM allocation threshold where preallocation does not pay off (128MB).
constexpr size_t L0UsmPreAllocThreshold = (128 << 20);
/// Host USM allocation threshold where preallocation does not pay off (8MB).
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 95ca54b062053..9936cfd77fbbf 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -463,6 +463,8 @@ class L0DeviceTy final : public GenericDeviceTy {
return L0Context.getEventPool().releaseEventObject(EventObj);
}
+ StagingBufferTy &getStagingBuffer() { return L0Context.getStagingBuffer(); }
+
bool supportsLargeMem() const { return L0Context.supportsLargeMem(); }
/// Returns the Queue from an async info object, or creates a new one if
diff --git a/offload/plugins-nextgen/level_zero/include/L0Memory.h b/offload/plugins-nextgen/level_zero/include/L0Memory.h
index 4f0ed23ba1215..8f2f3422b770e 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Memory.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Memory.h
@@ -463,6 +463,8 @@ class StagingBufferTy {
return Plugin::success();
}
+ bool initialized() const { return Context != nullptr; }
+
void init(ze_context_handle_t ContextIn, size_t SizeIn, size_t CountIn) {
Context = ContextIn;
Size = SizeIn;
@@ -471,6 +473,13 @@ class StagingBufferTy {
void reset() { Offset = 0; }
+ /// Always return the first buffer.
+ Expected<void *> get() {
+ if (Size == 0 || Count == 0)
+ return nullptr;
+ return Buffers.empty() ? addBuffers() : Buffers.front();
+ }
+
/// Return the next available buffer.
Expected<void *> getNext() {
void *Ret = nullptr;
@@ -494,6 +503,9 @@ class StagingBufferTy {
Offset += Size;
return Ret;
}
+
+ /// Return either a fixed buffer or next buffer.
+ Expected<void *> get(bool Next) { return Next ? getNext() : get(); }
};
} // namespace llvm::omp::target::plugin
diff --git a/offload/plugins-nextgen/level_zero/include/L0Plugin.h b/offload/plugins-nextgen/level_zero/include/L0Plugin.h
index e48e689b44282..56a708b7609a4 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Plugin.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Plugin.h
@@ -55,6 +55,9 @@ class LevelZeroPluginTy final : public GenericPluginTy {
/// Context (and Driver) specific data.
std::list<L0ContextTy> ContextList;
+ // Table containing per-thread information for each Context using TLS.
+ L0ContextTLSTableTy ContextTLSTable;
+
/// L0 plugin options.
L0OptionsTy Options;
@@ -66,6 +69,10 @@ class LevelZeroPluginTy final : public GenericPluginTy {
LevelZeroPluginTy() : GenericPluginTy(getTripleArch()) {}
virtual ~LevelZeroPluginTy() = default;
+ L0ContextTLSTy &getContextTLS(ze_context_handle_t Context) {
+ return ContextTLSTable.get(Context);
+ }
+
const L0OptionsTy &getOptions() { return Options; }
const L0DeviceTy &getDeviceFromId(int32_t DeviceId) const {
diff --git a/offload/plugins-nextgen/level_zero/include/L0Queue.h b/offload/plugins-nextgen/level_zero/include/L0Queue.h
index a1af2578a06a7..832994099f9e6 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Queue.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Queue.h
@@ -20,7 +20,6 @@
#include <tuple>
#include "L0CmdListManager.h"
-#include "L0Memory.h"
#include "L0Options.h"
namespace llvm::omp::target::plugin {
@@ -37,8 +36,6 @@ class L0QueueTy {
L0CmdListManagerTy *CmdList = nullptr;
/// Whether the queue is in-order or out-of-order.
bool CreateQueueInOrder;
- /// Host buffers that remain valid until this queue completes.
- StagingBufferTy StagingBuffer;
public:
L0QueueTy(L0DeviceTy &Device, bool IsInorder = true)
@@ -50,8 +47,8 @@ class L0QueueTy {
Error init();
Error deinit();
- Error synchronize();
- Expected<bool> hasPendingWork();
+ Error synchronize() { return synchronizeImpl(); }
+ Expected<bool> hasPendingWork() { return hasPendingWorkImpl(); }
Error memoryCopy(void *Dst, const void *Src, size_t Size) {
if (Size == 0)
diff --git a/offload/plugins-nextgen/level_zero/src/L0Context.cpp b/offload/plugins-nextgen/level_zero/src/L0Context.cpp
index 7809b4b24e192..be21a2fdc22ed 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Context.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Context.cpp
@@ -81,4 +81,14 @@ Error L0ContextTy::deinit() {
return Plugin::success();
}
+StagingBufferTy &L0ContextTy::getStagingBuffer() {
+ auto &TLS = Plugin.getContextTLS(getZeContext());
+ auto &Buffer = TLS.getStagingBuffer();
+ const auto &Options = Plugin.getOptions();
+ if (!Buffer.initialized())
+ Buffer.init(getZeContext(), Options.StagingBufferSize,
+ Options.StagingBufferCount);
+ return Buffer;
+}
+
} // namespace llvm::omp::target::plugin
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index 4b7c4bc7ce14e..c6cc578adb656 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -282,6 +282,7 @@ Error L0DeviceTy::synchronizeImpl(__tgt_async_info &AsyncInfo,
if (ReleaseQueue) {
releaseQueue(Queue);
+ getStagingBuffer().reset();
AsyncInfo.Queue = nullptr;
}
@@ -316,6 +317,7 @@ Error L0DeviceTy::queryAsyncImpl(__tgt_async_info &AsyncInfo, bool ReleaseQueue,
if (ReleaseQueue) {
releaseQueue(Queue);
+ getStagingBuffer().reset();
AsyncInfo.Queue = nullptr;
}
diff --git a/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp b/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
index d3c98579af33d..5723644c6763c 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Plugin.cpp
@@ -133,6 +133,8 @@ Expected<int32_t> LevelZeroPluginTy::initImpl() {
Error LevelZeroPluginTy::deinitImpl() {
ODBG(OLDT_Deinit) << "Deinit Level0 plugin!";
+ if (auto Err = ContextTLSTable.deinit())
+ return Err;
for (auto &Context : ContextList)
if (auto Err = Context.deinit())
return Err;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index 7e24d8eeae615..59cb73a29152a 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -18,6 +18,7 @@
#include "llvm/Support/MathExtras.h"
#include <algorithm>
+#include <vector>
namespace llvm::omp::target::plugin {
@@ -28,46 +29,22 @@ Error L0QueueTy::init() {
if (!CmdListOrErr)
return CmdListOrErr.takeError();
CmdList = *CmdListOrErr;
-
- const auto &Options = Device.getPlugin().getOptions();
- StagingBuffer.init(
- Device.getZeContext(),
- std::max(Options.StagingBufferSize, L0MaxMemFillPatternSize),
- Options.StagingBufferCount);
return initImpl();
}
Error L0QueueTy::deinit() {
- Error AllErrors = deinitImpl();
+ if (auto Err = deinitImpl())
+ return Err;
reset();
- if (auto Err = StagingBuffer.clear())
- AllErrors = joinErrors(std::move(AllErrors), std::move(Err));
-
if (CmdList)
if (auto Err = Device.releaseCmdListManager(CmdList))
- AllErrors = joinErrors(std::move(AllErrors), std::move(Err));
+ return Err;
CmdList = nullptr;
- return AllErrors;
-}
-
-Error L0QueueTy::synchronize() {
- if (auto Err = synchronizeImpl())
- return Err;
- StagingBuffer.reset();
return Plugin::success();
}
-Expected<bool> L0QueueTy::hasPendingWork() {
- auto PendingWorkOrErr = hasPendingWorkImpl();
- if (!PendingWorkOrErr)
- return PendingWorkOrErr.takeError();
- if (!*PendingWorkOrErr)
- StagingBuffer.reset();
- return *PendingWorkOrErr;
-}
-
Error L0QueueTy::dispatchLaunchKernel(ze_kernel_handle_t Kernel,
L0LaunchEnvTy &KEnv,
ze_event_handle_t SignalEvent,
@@ -87,11 +64,6 @@ Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
if (Size == 0 || PatternSize == 0)
return Plugin::success();
- if (PatternSize > L0MaxMemFillPatternSize)
- return Plugin::error(
- ErrorCode::INVALID_SIZE,
- "memory fill pattern size exceeds the 1024-byte maximum");
-
if (llvm::isPowerOf2_64(PatternSize) && (Size % PatternSize == 0) &&
PatternSize <= Device.getMaxMemFillPatternSize()) {
// Native L0 memory fill is possible directly.
@@ -131,20 +103,24 @@ Error L0QueueTy::memoryFillReplicateImpl(void *Ptr, const void *Pattern,
size_t PatternSize, size_t Size) {
auto *Dst = static_cast<unsigned char *>(Ptr);
- auto SeedOrErr = StagingBuffer.getNext();
- if (!SeedOrErr)
- return SeedOrErr.takeError();
- auto *Seed = static_cast<unsigned char *>(*SeedOrErr);
- if (!Seed)
- return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
- "failed to allocate a memory fill staging buffer");
-
- // Keep the seed pattern-aligned so each subsequent copy preserves it.
- const size_t SeedLimit = std::min(Size, L0MaxMemFillPatternSize);
+ // Extend small patterns to avoid several inefficient device copies.
+ constexpr size_t MinExtendedSeedSize = 1024;
+ const size_t SeedLimit =
+ std::min(Size, std::max(PatternSize, MinExtendedSeedSize));
size_t BytesFilled = (SeedLimit / PatternSize) * PatternSize;
- extendPattern(Seed, BytesFilled, Pattern, PatternSize);
+ std::vector<unsigned char> Seed(BytesFilled);
+ extendPattern(Seed.data(), BytesFilled, Pattern, PatternSize);
+
+ const auto TgtType = Device.getMemAllocType(Ptr);
+ if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
+ if (auto Err = synchronize())
+ return Err;
+
+ if (auto Err = dataSubmit(Dst, Seed.data(), BytesFilled))
+ return Err;
- if (auto Err = memoryCopy(Dst, Seed, BytesFilled))
+ // Ensure the asynchronous submission no longer references the host seed.
+ if (auto Err = synchronize())
return Err;
// Clone the seed, doubling each time, until it fills the entire destination.
@@ -280,7 +256,7 @@ Error L0AsyncQueueTy::dataRetrieveImpl(void *HstPtr, const void *TgtPtr,
static_cast<size_t>(Size) <=
Device.getPlugin().getOptions().StagingBufferSize &&
Device.getMemAllocType(HstPtr) != ZE_MEMORY_TYPE_HOST) {
- auto PtrOrErr = StagingBuffer.getNext();
+ auto PtrOrErr = Device.getStagingBuffer().get(/*IsAsync*/ true);
if (!PtrOrErr)
return PtrOrErr.takeError();
DstPtr = *PtrOrErr;
@@ -311,7 +287,7 @@ Error L0AsyncQueueTy::dataSubmitImpl(void *TgtPtr, const void *HstPtr,
static_cast<size_t>(Size) <=
Device.getPlugin().getOptions().StagingBufferSize &&
Device.getMemAllocType(HstPtr) != ZE_MEMORY_TYPE_HOST) {
- auto PtrOrErr = StagingBuffer.getNext();
+ auto PtrOrErr = Device.getStagingBuffer().get(/*IsAsync*/ true);
if (!PtrOrErr)
return PtrOrErr.takeError();
SrcPtr = *PtrOrErr;
diff --git a/offload/unittests/OffloadAPI/memory/olMemFill.cpp b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
index 3ef3e0072e3a0..3437b78a9548c 100644
--- a/offload/unittests/OffloadAPI/memory/olMemFill.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
@@ -210,15 +210,6 @@ TEST_P(olMemFillTest, InvalidPatternSizeLargerThanFillSize) {
olMemFree(Alloc);
}
-TEST(olMemFillValidationTest, InvalidPatternSizeTooLarge) {
- constexpr size_t Size = 1025;
- std::array<unsigned char, Size> Alloc{};
- std::array<unsigned char, Size> Pattern{};
- ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
- olMemFill(nullptr, Alloc.data(), Pattern.size(), Pattern.data(),
- Alloc.size()));
-}
-
// Even though L0, CUDA and HSA do not support non-power-of-two patterns,
// plugins are currently expected to handle arbitrary pattern sizes.
// The following tests are intended to cover the fallback paths
>From 8a06de2ce6343d9db3d39c59d1e7f69364e1f8b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 13 Aug 2026 10:46:57 +0000
Subject: [PATCH 3/6] cleanup extendPattern, correct comments
---
.../level_zero/src/L0Queue.cpp | 38 +++++++++++--------
1 file changed, 23 insertions(+), 15 deletions(-)
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index 59cb73a29152a..51a7dd567dfbe 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -90,13 +90,22 @@ Error L0QueueTy::memoryFillFallbackImpl(void *Ptr, const void *Pattern,
return memoryFillReplicateImpl(Ptr, Pattern, PatternSize, Size);
}
-static void extendPattern(unsigned char *Dst, size_t Size, const void *Pattern,
- size_t PatternSize) {
- assert(Size >= PatternSize && Size % PatternSize == 0 &&
- "Invalid pattern extension size");
- std::copy_n(static_cast<const unsigned char *>(Pattern), PatternSize, Dst);
- for (size_t Offset = PatternSize; Offset < Size; ++Offset)
- Dst[Offset] = Dst[Offset - PatternSize];
+/// Construct a seed by repeating \p Pattern. When \p PatternSize is at most
+/// \p MinSize, the seed size is a multiple of \p PatternSize in the range
+/// [MinSize, 2 * MinSize). Otherwise, return a copy of \p Pattern.
+static std::vector<unsigned char>
+extendPattern(const void *Pattern, size_t PatternSize, size_t MinSize) {
+ assert(PatternSize > 0 && MinSize > 0 && "Invalid pattern extension size");
+ const auto *PatternBytes = static_cast<const unsigned char *>(Pattern);
+ if (PatternSize > MinSize)
+ return std::vector<unsigned char>(PatternBytes, PatternBytes + PatternSize);
+
+ const size_t NumPatterns = (MinSize + PatternSize - 1) / PatternSize;
+ std::vector<unsigned char> Seed(NumPatterns * PatternSize);
+ std::copy_n(PatternBytes, PatternSize, Seed.begin());
+ for (size_t Offset = PatternSize; Offset < Seed.size(); ++Offset)
+ Seed[Offset] = Seed[Offset - PatternSize];
+ return Seed;
}
Error L0QueueTy::memoryFillReplicateImpl(void *Ptr, const void *Pattern,
@@ -104,22 +113,21 @@ Error L0QueueTy::memoryFillReplicateImpl(void *Ptr, const void *Pattern,
auto *Dst = static_cast<unsigned char *>(Ptr);
// Extend small patterns to avoid several inefficient device copies.
- constexpr size_t MinExtendedSeedSize = 1024;
- const size_t SeedLimit =
- std::min(Size, std::max(PatternSize, MinExtendedSeedSize));
- size_t BytesFilled = (SeedLimit / PatternSize) * PatternSize;
- std::vector<unsigned char> Seed(BytesFilled);
- extendPattern(Seed.data(), BytesFilled, Pattern, PatternSize);
+ const auto Seed = extendPattern(Pattern, PatternSize, /*MinSize=*/1024);
+ size_t BytesFilled = std::min(Seed.size(), Size);
const auto TgtType = Device.getMemAllocType(Ptr);
- if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
+ // dataSubmit() writes host/shared destinations directly, so complete earlier
+ // queue work before modifying the destination from the host.
+ if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED) {
if (auto Err = synchronize())
return Err;
+ }
if (auto Err = dataSubmit(Dst, Seed.data(), BytesFilled))
return Err;
- // Ensure the asynchronous submission no longer references the host seed.
+ // Complete the seed submission before its host storage goes out of scope.
if (auto Err = synchronize())
return Err;
>From 80ce43d61dbc98a9e42df8425d7ed854e2c21d28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 13 Aug 2026 10:57:23 +0000
Subject: [PATCH 4/6] clarify no async guarantee in api
---
offload/liboffload/API/Memory.td | 2 +-
.../unittests/OffloadAPI/memory/olMemFill.cpp | 20 -------------------
2 files changed, 1 insertion(+), 21 deletions(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 71dd08c377dda..6c771d89f58af 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -208,7 +208,7 @@ def olMemcpy : Function {
def olMemFill : Function {
let desc = "Fill memory with copies of the given pattern";
let details = [
- "Filling with patterns of a different size than 1, 2 or 4 bytes may be less performant",
+ "Filling with patterns of a different size than 1, 2 or 4 may be less performant and may synchronize the queue",
"The destination pointer and queue must be associated with the same device",
"The fill size must be a multiple of the pattern size",
];
diff --git a/offload/unittests/OffloadAPI/memory/olMemFill.cpp b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
index 3437b78a9548c..e76bbd5945ff5 100644
--- a/offload/unittests/OffloadAPI/memory/olMemFill.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
@@ -233,26 +233,6 @@ TEST_P(olMemFillTest, SuccessNonPow2PatternManaged) {
olMemFree(Alloc);
}
-TEST_P(olMemFillTest, SuccessNonPow2PatternManagedEnqueue) {
- constexpr size_t Size = FallbackPattern.size() * 1000;
- ManuallyTriggeredTask Manual;
- ASSERT_SUCCESS(Manual.enqueue(Queue));
-
- void *Alloc;
- ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
-
- ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
- FallbackPattern.data(), Size));
- ASSERT_SUCCESS(Manual.trigger());
- olSyncQueue(Queue);
-
- auto *AllocPtr = reinterpret_cast<unsigned char *>(Alloc);
- for (size_t I = 0; I < Size; I++)
- ASSERT_EQ(AllocPtr[I], FallbackPattern[I % FallbackPattern.size()]);
-
- olMemFree(Alloc);
-}
-
TEST_P(olMemFillTest, SuccessNonPow2PatternDevice) {
constexpr size_t Size = FallbackPattern.size() * 1000;
void *Alloc;
>From 46829f0553642e07ca374ac504fee326c89febe5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 13 Aug 2026 11:17:03 +0000
Subject: [PATCH 5/6] bytes
---
offload/liboffload/API/Memory.td | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 6c771d89f58af..17eabeccce9e0 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -208,7 +208,7 @@ def olMemcpy : Function {
def olMemFill : Function {
let desc = "Fill memory with copies of the given pattern";
let details = [
- "Filling with patterns of a different size than 1, 2 or 4 may be less performant and may synchronize the queue",
+ "Filling with patterns of a different size than 1, 2 or 4 bytes may be less performant and may synchronize the queue",
"The destination pointer and queue must be associated with the same device",
"The fill size must be a multiple of the pattern size",
];
>From d1b3f57e882a1f173e390f7a6b7689e13dab3488 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 20 Aug 2026 12:07:08 +0000
Subject: [PATCH 6/6] sync fills in L0SyncQueueTy, use replication fallback on
discrete devices
---
offload/plugins-nextgen/level_zero/include/L0Queue.h | 2 ++
offload/plugins-nextgen/level_zero/src/L0Queue.cpp | 10 +++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/offload/plugins-nextgen/level_zero/include/L0Queue.h b/offload/plugins-nextgen/level_zero/include/L0Queue.h
index 832994099f9e6..13a4a1e64c3c9 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Queue.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Queue.h
@@ -264,6 +264,8 @@ class L0SyncQueueTy : public L0InorderQueueTy {
Error launchKernelImpl(ze_kernel_handle_t Kernel,
L0LaunchEnvTy &KEnv) override;
Error hostCallImpl(void (*Callback)(void *), void *UserData) override;
+ Error memoryFillImpl(void *Ptr, const void *Pattern, size_t PatternSize,
+ size_t Size) override;
Error memoryFillFallbackImpl(void *Ptr, const void *Pattern,
size_t PatternSize, size_t Size) override;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index 51a7dd567dfbe..69fc3bd61db1b 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -445,10 +445,18 @@ Error L0SyncQueueTy::hostCallImpl(void (*Callback)(void *), void *UserData) {
return CmdList->hostSynchronize();
}
+Error L0SyncQueueTy::memoryFillImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ if (auto Err = L0QueueTy::memoryFillImpl(Ptr, Pattern, PatternSize, Size))
+ return Err;
+ return CmdList->hostSynchronize();
+}
+
Error L0SyncQueueTy::memoryFillFallbackImpl(void *Ptr, const void *Pattern,
size_t PatternSize, size_t Size) {
const auto TgtType = Device.getMemAllocType(Ptr);
- if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
+ if (TgtType == ZE_MEMORY_TYPE_HOST ||
+ (TgtType == ZE_MEMORY_TYPE_SHARED && !Device.isDiscreteDevice()))
return memoryFillHostImpl(Ptr, Pattern, PatternSize, Size);
return L0QueueTy::memoryFillFallbackImpl(Ptr, Pattern, PatternSize, Size);
}
More information about the llvm-commits
mailing list