[llvm] [offload] support arbitrary memoryFill pattern sizes in L0 plugin (PR #209463)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 05:34:27 PDT 2026
https://github.com/311Volt created https://github.com/llvm/llvm-project/pull/209463
None
>From 63048a4f4cc0a3a8d28495df993271bf8eaf870a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Tue, 14 Jul 2026 12:33:32 +0000
Subject: [PATCH] [offload] support arbitrary memoryFill pattern sizes in L0
plugin
---
offload/liboffload/API/Memory.td | 2 +-
.../level_zero/include/L0Device.h | 33 ++++--
.../level_zero/include/L0Queue.h | 17 ++-
.../level_zero/src/L0Device.cpp | 31 +++---
.../level_zero/src/L0Queue.cpp | 103 ++++++++++++++++++
.../unittests/OffloadAPI/memory/olMemFill.cpp | 81 ++++++++++++++
6 files changed, 241 insertions(+), 26 deletions(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 9459270eb9b16..3447e3587e38e 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -157,7 +157,7 @@ def olMemcpy : Function {
def olMemFill : Function {
let desc = "Fill memory with copies of the given pattern";
let details = [
- "Filling with patterns larger than 4 bytes may be less performant",
+ "Filling with patterns of a different size than 1, 2 or 4 bytes may be less performant",
"The destination pointer and queue must be associated with the same device",
"The fill size must be a multiple of the pattern size",
];
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 408fb452a0dc8..313bdf1ff7862 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -22,7 +22,9 @@
#include "L0Program.h"
#include "L0Queue.h"
#include "PluginInterface.h"
+#include <cstdint>
#include <limits>
+#include <optional>
namespace llvm::omp::target::plugin {
@@ -75,6 +77,17 @@ struct L0DeviceIdTy {
: zeId(Device), RootId(RootId), SubId(SubId), CCSId(CCSId) {}
};
+/// Properties of the compute command queue group selected for a device.
+struct ComputeGroupInfoTy {
+ /// Command queue group ordinal.
+ uint32_t Ordinal = std::numeric_limits<uint32_t>::max();
+ /// Number of queues in the group.
+ uint32_t NumQueues = 0;
+ /// Maximum pattern size accepted by zeCommandListMemoryFill for this device.
+ /// 0 means value is unavailable.
+ size_t MaxMemFillPatternSize = 0;
+};
+
class L0DeviceTy final : public GenericDeviceTy {
// Level Zero Context for this Device.
L0ContextTy &l0Context;
@@ -104,10 +117,9 @@ class L0DeviceTy final : public GenericDeviceTy {
/// L0 Device ID as string.
std::string zeId;
- /// Command queue group ordinals for each device.
- static constexpr uint32_t MaxOrdinal = std::numeric_limits<uint32_t>::max();
-
- std::pair<uint32_t, uint32_t> ComputeOrdinal{MaxOrdinal, 0};
+ /// Compute command queue group info for this device. Value is unspecified
+ /// unless the device reached a valid initialized state.
+ ComputeGroupInfoTy ComputeGroupInfo;
/// Command queue index for each device.
uint32_t ComputeIndex = 0;
@@ -133,8 +145,9 @@ class L0DeviceTy final : public GenericDeviceTy {
DeviceArchTy computeArch() const;
- /// Get default compute group ordinal. Returns Ordinal-NumQueues pair.
- std::pair<uint32_t, uint32_t> findComputeOrdinal();
+ /// Find the default compute command queue group. Returns std::nullopt if
+ /// the device exposes no compute queue group.
+ std::optional<ComputeGroupInfoTy> findCommandQueueGroup();
/// Helper function to call global constructors or destructors.
Error callGlobalCtorDtorCommon(GenericPluginTy &Plugin, DeviceImageTy &Image,
@@ -338,8 +351,12 @@ class L0DeviceTy final : public GenericDeviceTy {
const std::string_view getUuid() const { return DeviceUuid; }
- uint32_t getComputeEngine() const { return ComputeOrdinal.first; }
- uint32_t getNumComputeQueues() const { return ComputeOrdinal.second; }
+ uint32_t getComputeEngine() const { return ComputeGroupInfo.Ordinal; }
+ uint32_t getNumComputeQueues() const { return ComputeGroupInfo.NumQueues; }
+
+ size_t getMaxMemFillPatternSize() {
+ return ComputeGroupInfo.MaxMemFillPatternSize;
+ }
void reportDeviceInfo() const;
diff --git a/offload/plugins-nextgen/level_zero/include/L0Queue.h b/offload/plugins-nextgen/level_zero/include/L0Queue.h
index ca477376fc0a6..2957ef558c847 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Queue.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Queue.h
@@ -69,10 +69,11 @@ class L0QueueTy {
return dataSubmitImpl(TgtPtr, HstPtr, Size);
}
+ // Enqueue a memory fill command. Supports arbitrary pattern sizes, including
+ // non-power-of-two sizes, by falling back to a less performant software fill
+ // if necessary.
Error memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
- size_t Size) {
- return memoryFillImpl(Ptr, Pattern, PatternSize, Size);
- }
+ size_t Size);
Error memoryPrefetch(const void *Ptr, size_t Size) {
if (Size == 0)
@@ -153,6 +154,16 @@ class L0QueueTy {
virtual Error appendWaitOnEventImpl(ze_event_handle_t Event) {
return CmdList->appendWaitOnEvent(Event);
}
+
+private:
+ /// Fallback fill for host-accessible target memory: replicate the pattern
+ /// directly on the host with std::copy_n.
+ Error memoryFillHostImpl(void *Ptr, const void *Pattern, size_t PatternSize,
+ size_t Size);
+ /// Fallback fill for non-host-accessible target memory: seed the pattern
+ /// once and grow the filled region via device copies, doubling each time.
+ Error memoryFillReplicateImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size);
};
class L0AsyncQueueTy : public L0QueueTy {
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index e0dfe0596b5ce..5130cf147a8be 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -98,31 +98,29 @@ bool L0DeviceTy::isDeviceIPorNewer(uint32_t Version) const {
return IPVersion.ipVersion >= Version;
}
-/// Get default compute group ordinal. Returns Ordinal-NumQueues pair.
-std::pair<uint32_t, uint32_t> L0DeviceTy::findComputeOrdinal() {
- std::pair<uint32_t, uint32_t> Ordinal{MaxOrdinal, 0};
+/// Find the default compute command queue group. Returns std::nullopt if the
+/// device exposes no compute queue group.
+std::optional<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
uint32_t Count = 0;
const auto zeDevice = getZeDevice();
- CALL_ZE_RET(Ordinal, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
- nullptr);
+ CALL_ZE_RET(std::nullopt, zeDeviceGetCommandQueueGroupProperties, zeDevice,
+ &Count, nullptr);
ze_command_queue_group_properties_t Init{
ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES, nullptr, 0, 0, 0};
std::vector<ze_command_queue_group_properties_t> Properties(Count, Init);
- CALL_ZE_RET(Ordinal, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
- Properties.data());
+ CALL_ZE_RET(std::nullopt, zeDeviceGetCommandQueueGroupProperties, zeDevice,
+ &Count, Properties.data());
for (uint32_t I = 0; I < Count; I++) {
// TODO: add a separate set of ordinals for compute queue groups which
// support cooperative kernels.
if (Properties[I].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) {
- Ordinal.first = I;
- Ordinal.second = Properties[I].numQueues;
- break;
+ return ComputeGroupInfoTy{/*Ordinal=*/I,
+ /*NumQueues=*/Properties[I].numQueues,
+ Properties[I].maxMemoryFillPatternSize};
}
}
- if (Ordinal.first == MaxOrdinal)
- ODBG(OLDT_Device) << "Error: no command queues are found";
- return Ordinal;
+ return std::nullopt;
}
/// Check if device supports cooperative kernels by checking if any command
@@ -202,7 +200,12 @@ Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
uid += std::to_string(DeviceProperties.uuid.id[n]);
DeviceUuid = std::move(uid);
- ComputeOrdinal = findComputeOrdinal();
+ auto ComputeGroupInfoOpt = findCommandQueueGroup();
+ if (not ComputeGroupInfoOpt)
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "Device %d (%s) has no compute command queue group",
+ DeviceId, getNameCStr());
+ ComputeGroupInfo = *ComputeGroupInfoOpt;
QueueCache.setCommandMode(getPlugin().getOptions().CommandMode);
SupportsCooperativeKernels = checkCooperativeKernelSupport();
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index 81cef7be2df48..5373aed37961e 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -15,6 +15,10 @@
#include "L0Kernel.h"
#include "L0Plugin.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/MathExtras.h"
+
+#include <algorithm>
+#include <vector>
namespace llvm::omp::target::plugin {
@@ -58,6 +62,105 @@ Error L0QueueTy::dispatchLaunchKernel(ze_kernel_handle_t Kernel,
KEnv.IsCooperative);
}
+static bool allBytesEqual(const unsigned char *buf, size_t size) {
+ if (size == 0)
+ return true;
+ unsigned char FirstByte = buf[0];
+ for (size_t i = 1; i < size; ++i) {
+ if (buf[i] != FirstByte)
+ return false;
+ }
+ return true;
+}
+
+Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
+ size_t Size) {
+ if (Size == 0 || PatternSize == 0)
+ return Plugin::success();
+
+ if (llvm::isPowerOf2_64(PatternSize) &&
+ PatternSize <= Device.getMaxMemFillPatternSize()) {
+ // Native L0 memory fill is possible directly.
+ return memoryFillImpl(Ptr, Pattern, PatternSize, Size);
+ }
+
+ const auto *PatternBytes = static_cast<const unsigned char *>(Pattern);
+ if (allBytesEqual(PatternBytes, PatternSize)) {
+ // All pattern bytes equal, substutition of 1 as PatternSize is equivalent,
+ // so native L0 memory fill is still possible.
+ return memoryFillImpl(Ptr, Pattern, 1, Size);
+ }
+
+ // TODO: if we insist on plugins supporting arbitrary pattern sizes, extra
+ // detection of repeating power-of-two patterns could be added here to allow
+ // native L0 memory fill for those cases as well.
+
+ // Native L0 fill cannot handle this pattern size, but target memory is
+ // host-accessible, so fall back to a software fill.
+ const auto TgtType = Device.getMemAllocType(Ptr);
+ if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
+ return memoryFillHostImpl(Ptr, Pattern, PatternSize, Size);
+
+ // We know at this point that TgtType == ZE_MEMORY_TYPE_DEVICE.
+ // Native fill and software fill are both impossible.
+ // Seed the pattern once and grow the filled region with device copies,
+ // doubling the amount copied each time.
+ return memoryFillReplicateImpl(Ptr, Pattern, PatternSize, Size);
+}
+
+Error L0QueueTy::memoryFillHostImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ auto *Dst = static_cast<unsigned char *>(Ptr);
+ const auto *Pat = static_cast<const unsigned char *>(Pattern);
+ for (size_t Offset = 0; Offset < Size;) {
+ const size_t Chunk = std::min(PatternSize, Size - Offset);
+ std::copy_n(Pat, Chunk, Dst + Offset);
+ Offset += Chunk;
+ }
+ return Plugin::success();
+}
+
+/// Replicate the pattern in \p Buf (of \p Size bytes) on the host until it is
+/// at least \p MinExtendedSize bytes long. The result is
+/// never larger than max(Size, 2 * MinExtendedSize).
+static std::vector<unsigned char> extendPattern(unsigned char *Buf, size_t Size,
+ size_t MinExtendedSize) {
+ assert(Size > 0 && MinExtendedSize > 0 &&
+ "Invalid pattern size or extension size");
+ const size_t NumPatterns =
+ std::max(static_cast<size_t>(1), (MinExtendedSize + Size - 1) / Size);
+ std::vector<unsigned char> Extended(NumPatterns * Size);
+ for (size_t i = 0; i < NumPatterns; i++)
+ std::copy_n(Buf, Size, Extended.begin() + i * Size);
+ return Extended;
+}
+
+Error L0QueueTy::memoryFillReplicateImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ auto *Dst = static_cast<unsigned char *>(Ptr);
+
+ // Grow the pattern on the host first - avoids several inefficient small
+ // device copies.
+ constexpr size_t MinExtendedSeedSize = 1024;
+ const auto ExtendedPattern =
+ extendPattern(static_cast<unsigned char *>(const_cast<void *>(Pattern)),
+ PatternSize, std::min(Size, MinExtendedSeedSize));
+
+ // Seed the (extended) pattern once using dataSubmit.
+ size_t BytesFilled = std::min(ExtendedPattern.size(), Size);
+ if (auto Err = dataSubmit(Dst, ExtendedPattern.data(), BytesFilled))
+ return Err;
+
+ // Clone the seed, doubling each time, until it fills the entire destination.
+ while (BytesFilled < Size) {
+ const size_t CopyChunkSize = std::min(BytesFilled, Size - BytesFilled);
+ if (auto Err = memoryCopy(Dst + BytesFilled, Dst, CopyChunkSize))
+ return Err;
+ BytesFilled += CopyChunkSize;
+ }
+ return Plugin::success();
+}
+
// L0AsyncQueueTy implementation.
Error L0AsyncQueueTy::deinitImpl() {
diff --git a/offload/unittests/OffloadAPI/memory/olMemFill.cpp b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
index b094c6298e16d..0f0170a12d346 100644
--- a/offload/unittests/OffloadAPI/memory/olMemFill.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
@@ -8,7 +8,9 @@
#include "../common/Fixtures.hpp"
#include <OffloadAPI.h>
+#include <array>
#include <gtest/gtest.h>
+#include <vector>
struct olMemFillTest : OffloadQueueTest {
void SetUp() override { RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp()); }
@@ -194,3 +196,82 @@ TEST_P(olMemFillTest, InvalidPatternSize) {
olSyncQueue(Queue);
olMemFree(Alloc);
}
+
+// Even though L0, CUDA and HSA do not support non-power-of-two patterns,
+// plugins are currently expected to handle arbitrary pattern sizes.
+// The following tests are intended to cover the fallback paths
+// for non-power-of-two patterns.
+static constexpr std::array<unsigned char, 3> FallbackPattern = {0x11, 0x22,
+ 0x33};
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternManaged) {
+ constexpr size_t Size = FallbackPattern.size() * 1000;
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+ olSyncQueue(Queue);
+
+ auto *AllocPtr = reinterpret_cast<unsigned char *>(Alloc);
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(AllocPtr[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternManagedEnqueue) {
+ constexpr size_t Size = FallbackPattern.size() * 1000;
+ ManuallyTriggeredTask Manual;
+ ASSERT_SUCCESS(Manual.enqueue(Queue));
+
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+ ASSERT_SUCCESS(Manual.trigger());
+ olSyncQueue(Queue);
+
+ auto *AllocPtr = reinterpret_cast<unsigned char *>(Alloc);
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(AllocPtr[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternDevice) {
+ constexpr size_t Size = FallbackPattern.size() * 1000;
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+
+ std::vector<unsigned char> HostBuf(Size);
+ ASSERT_SUCCESS(olMemcpy(Queue, HostBuf.data(), Host, Alloc, Device, Size));
+ olSyncQueue(Queue);
+
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(HostBuf[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternDeviceSmall) {
+ constexpr size_t Size = FallbackPattern.size() * 2;
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+
+ std::vector<unsigned char> HostBuf(Size);
+ ASSERT_SUCCESS(olMemcpy(Queue, HostBuf.data(), Host, Alloc, Device, Size));
+ olSyncQueue(Queue);
+
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(HostBuf[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
More information about the llvm-commits
mailing list