[llvm] [offload] support arbitrary memoryFill pattern sizes in L0 plugin (PR #209724)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 03:46:30 PDT 2026
Jan =?utf-8?q?Trusiłło?= <jan.trusillo at intel.com>,
Jan =?utf-8?q?Trusiłło?= <jan.trusillo at intel.com>,
Jan =?utf-8?q?Trusiłło?= <jan.trusillo at intel.com>,
Jan =?utf-8?q?Trusiłło?= <jan.trusillo at intel.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/209724 at github.com>
https://github.com/311Volt updated https://github.com/llvm/llvm-project/pull/209724
>From 5858ad68d6dd53105255a7a262ae3d4dc967442e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Tue, 14 Jul 2026 12:33:32 +0000
Subject: [PATCH 1/5] [offload] support arbitrary memoryFill pattern sizes in
L0 plugin
---
offload/liboffload/API/Memory.td | 2 +-
.../level_zero/include/L0Device.h | 33 ++++--
.../level_zero/include/L0Queue.h | 17 ++-
.../level_zero/src/L0Device.cpp | 31 +++---
.../level_zero/src/L0Queue.cpp | 105 ++++++++++++++++++
.../unittests/OffloadAPI/memory/olMemFill.cpp | 81 ++++++++++++++
6 files changed, 243 insertions(+), 26 deletions(-)
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 9459270eb9b16..3447e3587e38e 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -157,7 +157,7 @@ def olMemcpy : Function {
def olMemFill : Function {
let desc = "Fill memory with copies of the given pattern";
let details = [
- "Filling with patterns larger than 4 bytes may be less performant",
+ "Filling with patterns of a different size than 1, 2 or 4 bytes may be less performant",
"The destination pointer and queue must be associated with the same device",
"The fill size must be a multiple of the pattern size",
];
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 408fb452a0dc8..313bdf1ff7862 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -22,7 +22,9 @@
#include "L0Program.h"
#include "L0Queue.h"
#include "PluginInterface.h"
+#include <cstdint>
#include <limits>
+#include <optional>
namespace llvm::omp::target::plugin {
@@ -75,6 +77,17 @@ struct L0DeviceIdTy {
: zeId(Device), RootId(RootId), SubId(SubId), CCSId(CCSId) {}
};
+/// Properties of the compute command queue group selected for a device.
+struct ComputeGroupInfoTy {
+ /// Command queue group ordinal.
+ uint32_t Ordinal = std::numeric_limits<uint32_t>::max();
+ /// Number of queues in the group.
+ uint32_t NumQueues = 0;
+ /// Maximum pattern size accepted by zeCommandListMemoryFill for this device.
+ /// 0 means value is unavailable.
+ size_t MaxMemFillPatternSize = 0;
+};
+
class L0DeviceTy final : public GenericDeviceTy {
// Level Zero Context for this Device.
L0ContextTy &l0Context;
@@ -104,10 +117,9 @@ class L0DeviceTy final : public GenericDeviceTy {
/// L0 Device ID as string.
std::string zeId;
- /// Command queue group ordinals for each device.
- static constexpr uint32_t MaxOrdinal = std::numeric_limits<uint32_t>::max();
-
- std::pair<uint32_t, uint32_t> ComputeOrdinal{MaxOrdinal, 0};
+ /// Compute command queue group info for this device. Value is unspecified
+ /// unless the device reached a valid initialized state.
+ ComputeGroupInfoTy ComputeGroupInfo;
/// Command queue index for each device.
uint32_t ComputeIndex = 0;
@@ -133,8 +145,9 @@ class L0DeviceTy final : public GenericDeviceTy {
DeviceArchTy computeArch() const;
- /// Get default compute group ordinal. Returns Ordinal-NumQueues pair.
- std::pair<uint32_t, uint32_t> findComputeOrdinal();
+ /// Find the default compute command queue group. Returns std::nullopt if
+ /// the device exposes no compute queue group.
+ std::optional<ComputeGroupInfoTy> findCommandQueueGroup();
/// Helper function to call global constructors or destructors.
Error callGlobalCtorDtorCommon(GenericPluginTy &Plugin, DeviceImageTy &Image,
@@ -338,8 +351,12 @@ class L0DeviceTy final : public GenericDeviceTy {
const std::string_view getUuid() const { return DeviceUuid; }
- uint32_t getComputeEngine() const { return ComputeOrdinal.first; }
- uint32_t getNumComputeQueues() const { return ComputeOrdinal.second; }
+ uint32_t getComputeEngine() const { return ComputeGroupInfo.Ordinal; }
+ uint32_t getNumComputeQueues() const { return ComputeGroupInfo.NumQueues; }
+
+ size_t getMaxMemFillPatternSize() {
+ return ComputeGroupInfo.MaxMemFillPatternSize;
+ }
void reportDeviceInfo() const;
diff --git a/offload/plugins-nextgen/level_zero/include/L0Queue.h b/offload/plugins-nextgen/level_zero/include/L0Queue.h
index ca477376fc0a6..2957ef558c847 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Queue.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Queue.h
@@ -69,10 +69,11 @@ class L0QueueTy {
return dataSubmitImpl(TgtPtr, HstPtr, Size);
}
+ // Enqueue a memory fill command. Supports arbitrary pattern sizes, including
+ // non-power-of-two sizes, by falling back to a less performant software fill
+ // if necessary.
Error memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
- size_t Size) {
- return memoryFillImpl(Ptr, Pattern, PatternSize, Size);
- }
+ size_t Size);
Error memoryPrefetch(const void *Ptr, size_t Size) {
if (Size == 0)
@@ -153,6 +154,16 @@ class L0QueueTy {
virtual Error appendWaitOnEventImpl(ze_event_handle_t Event) {
return CmdList->appendWaitOnEvent(Event);
}
+
+private:
+ /// Fallback fill for host-accessible target memory: replicate the pattern
+ /// directly on the host with std::copy_n.
+ Error memoryFillHostImpl(void *Ptr, const void *Pattern, size_t PatternSize,
+ size_t Size);
+ /// Fallback fill for non-host-accessible target memory: seed the pattern
+ /// once and grow the filled region via device copies, doubling each time.
+ Error memoryFillReplicateImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size);
};
class L0AsyncQueueTy : public L0QueueTy {
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index e0dfe0596b5ce..5130cf147a8be 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -98,31 +98,29 @@ bool L0DeviceTy::isDeviceIPorNewer(uint32_t Version) const {
return IPVersion.ipVersion >= Version;
}
-/// Get default compute group ordinal. Returns Ordinal-NumQueues pair.
-std::pair<uint32_t, uint32_t> L0DeviceTy::findComputeOrdinal() {
- std::pair<uint32_t, uint32_t> Ordinal{MaxOrdinal, 0};
+/// Find the default compute command queue group. Returns std::nullopt if the
+/// device exposes no compute queue group.
+std::optional<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
uint32_t Count = 0;
const auto zeDevice = getZeDevice();
- CALL_ZE_RET(Ordinal, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
- nullptr);
+ CALL_ZE_RET(std::nullopt, zeDeviceGetCommandQueueGroupProperties, zeDevice,
+ &Count, nullptr);
ze_command_queue_group_properties_t Init{
ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES, nullptr, 0, 0, 0};
std::vector<ze_command_queue_group_properties_t> Properties(Count, Init);
- CALL_ZE_RET(Ordinal, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
- Properties.data());
+ CALL_ZE_RET(std::nullopt, zeDeviceGetCommandQueueGroupProperties, zeDevice,
+ &Count, Properties.data());
for (uint32_t I = 0; I < Count; I++) {
// TODO: add a separate set of ordinals for compute queue groups which
// support cooperative kernels.
if (Properties[I].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) {
- Ordinal.first = I;
- Ordinal.second = Properties[I].numQueues;
- break;
+ return ComputeGroupInfoTy{/*Ordinal=*/I,
+ /*NumQueues=*/Properties[I].numQueues,
+ Properties[I].maxMemoryFillPatternSize};
}
}
- if (Ordinal.first == MaxOrdinal)
- ODBG(OLDT_Device) << "Error: no command queues are found";
- return Ordinal;
+ return std::nullopt;
}
/// Check if device supports cooperative kernels by checking if any command
@@ -202,7 +200,12 @@ Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
uid += std::to_string(DeviceProperties.uuid.id[n]);
DeviceUuid = std::move(uid);
- ComputeOrdinal = findComputeOrdinal();
+ auto ComputeGroupInfoOpt = findCommandQueueGroup();
+ if (not ComputeGroupInfoOpt)
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "Device %d (%s) has no compute command queue group",
+ DeviceId, getNameCStr());
+ ComputeGroupInfo = *ComputeGroupInfoOpt;
QueueCache.setCommandMode(getPlugin().getOptions().CommandMode);
SupportsCooperativeKernels = checkCooperativeKernelSupport();
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index 81cef7be2df48..b3d1ddf48b7c2 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -15,6 +15,10 @@
#include "L0Kernel.h"
#include "L0Plugin.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/MathExtras.h"
+
+#include <algorithm>
+#include <vector>
namespace llvm::omp::target::plugin {
@@ -58,6 +62,107 @@ Error L0QueueTy::dispatchLaunchKernel(ze_kernel_handle_t Kernel,
KEnv.IsCooperative);
}
+static bool allBytesEqual(const unsigned char *buf, size_t size) {
+ if (size == 0)
+ return true;
+ unsigned char FirstByte = buf[0];
+ for (size_t i = 1; i < size; ++i) {
+ if (buf[i] != FirstByte)
+ return false;
+ }
+ return true;
+}
+
+Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
+ size_t Size) {
+ if (Size == 0 || PatternSize == 0)
+ return Plugin::success();
+
+ if (llvm::isPowerOf2_64(PatternSize) &&
+ PatternSize <= Device.getMaxMemFillPatternSize()) {
+ // Native L0 memory fill is possible directly.
+ return memoryFillImpl(Ptr, Pattern, PatternSize, Size);
+ }
+
+ const auto *PatternBytes = static_cast<const unsigned char *>(Pattern);
+ if (allBytesEqual(PatternBytes, PatternSize)) {
+ // All pattern bytes equal, substutition of 1 as PatternSize is equivalent,
+ // so native L0 memory fill is still possible.
+ return memoryFillImpl(Ptr, Pattern, 1, Size);
+ }
+
+ // TODO: if we insist on plugins supporting arbitrary pattern sizes, extra
+ // detection of repeating power-of-two patterns could be added here to allow
+ // native L0 memory fill for those cases as well.
+
+ // Native L0 fill cannot handle this pattern size, but target memory is
+ // host-accessible, so fall back to a software fill.
+ const auto TgtType = Device.getMemAllocType(Ptr);
+ if (TgtType == ZE_MEMORY_TYPE_HOST || TgtType == ZE_MEMORY_TYPE_SHARED)
+ return memoryFillHostImpl(Ptr, Pattern, PatternSize, Size);
+
+ // We know at this point that TgtType == ZE_MEMORY_TYPE_DEVICE.
+ // Native fill and software fill are both impossible.
+ // Seed the pattern once and grow the filled region with device copies,
+ // doubling the amount copied each time.
+ return memoryFillReplicateImpl(Ptr, Pattern, PatternSize, Size);
+}
+
+Error L0QueueTy::memoryFillHostImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ auto *Dst = static_cast<unsigned char *>(Ptr);
+ const auto *Pat = static_cast<const unsigned char *>(Pattern);
+ for (size_t Offset = 0; Offset < Size;) {
+ const size_t Chunk = std::min(PatternSize, Size - Offset);
+ std::copy_n(Pat, Chunk, Dst + Offset);
+ Offset += Chunk;
+ }
+ return Plugin::success();
+}
+
+/// Replicate the pattern in \p Buf (of \p Size bytes) on the host until it is
+/// at least \p MinExtendedSize bytes long. The result is
+/// never larger than max(Size, 2 * MinExtendedSize).
+static std::vector<unsigned char> extendPattern(unsigned char *Buf, size_t Size,
+ size_t MinExtendedSize) {
+ assert(Size > 0 && MinExtendedSize > 0 &&
+ "Invalid pattern size or extension size");
+ const size_t NumPatterns =
+ std::max(static_cast<size_t>(1), (MinExtendedSize + Size - 1) / Size);
+ std::vector<unsigned char> Extended(NumPatterns * Size);
+ for (size_t i = 0; i < NumPatterns; i++)
+ std::copy_n(Buf, Size, Extended.begin() + i * Size);
+ return Extended;
+}
+
+Error L0QueueTy::memoryFillReplicateImpl(void *Ptr, const void *Pattern,
+ size_t PatternSize, size_t Size) {
+ auto *Dst = static_cast<unsigned char *>(Ptr);
+
+ // Grow the pattern on the host first - avoids several inefficient small
+ // device copies.
+ constexpr size_t MinExtendedSeedSize = 1024;
+ const auto ExtendedPattern =
+ extendPattern(static_cast<unsigned char *>(const_cast<void *>(Pattern)),
+ PatternSize, std::min(Size, MinExtendedSeedSize));
+
+ // Seed the (extended) pattern once using dataSubmit.
+ size_t BytesFilled = std::min(ExtendedPattern.size(), Size);
+ if (auto Err = dataSubmit(Dst, ExtendedPattern.data(), BytesFilled))
+ return Err;
+
+ // Clone the seed, doubling each time, until it fills the entire destination.
+ while (BytesFilled < Size) {
+ if (auto Err = dataFence())
+ return Err;
+ const size_t CopyChunkSize = std::min(BytesFilled, Size - BytesFilled);
+ if (auto Err = memoryCopy(Dst + BytesFilled, Dst, CopyChunkSize))
+ return Err;
+ BytesFilled += CopyChunkSize;
+ }
+ return Plugin::success();
+}
+
// L0AsyncQueueTy implementation.
Error L0AsyncQueueTy::deinitImpl() {
diff --git a/offload/unittests/OffloadAPI/memory/olMemFill.cpp b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
index b094c6298e16d..0f0170a12d346 100644
--- a/offload/unittests/OffloadAPI/memory/olMemFill.cpp
+++ b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
@@ -8,7 +8,9 @@
#include "../common/Fixtures.hpp"
#include <OffloadAPI.h>
+#include <array>
#include <gtest/gtest.h>
+#include <vector>
struct olMemFillTest : OffloadQueueTest {
void SetUp() override { RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp()); }
@@ -194,3 +196,82 @@ TEST_P(olMemFillTest, InvalidPatternSize) {
olSyncQueue(Queue);
olMemFree(Alloc);
}
+
+// Even though L0, CUDA and HSA do not support non-power-of-two patterns,
+// plugins are currently expected to handle arbitrary pattern sizes.
+// The following tests are intended to cover the fallback paths
+// for non-power-of-two patterns.
+static constexpr std::array<unsigned char, 3> FallbackPattern = {0x11, 0x22,
+ 0x33};
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternManaged) {
+ constexpr size_t Size = FallbackPattern.size() * 1000;
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+ olSyncQueue(Queue);
+
+ auto *AllocPtr = reinterpret_cast<unsigned char *>(Alloc);
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(AllocPtr[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternManagedEnqueue) {
+ constexpr size_t Size = FallbackPattern.size() * 1000;
+ ManuallyTriggeredTask Manual;
+ ASSERT_SUCCESS(Manual.enqueue(Queue));
+
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+ ASSERT_SUCCESS(Manual.trigger());
+ olSyncQueue(Queue);
+
+ auto *AllocPtr = reinterpret_cast<unsigned char *>(Alloc);
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(AllocPtr[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternDevice) {
+ constexpr size_t Size = FallbackPattern.size() * 1000;
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+
+ std::vector<unsigned char> HostBuf(Size);
+ ASSERT_SUCCESS(olMemcpy(Queue, HostBuf.data(), Host, Alloc, Device, Size));
+ olSyncQueue(Queue);
+
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(HostBuf[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessNonPow2PatternDeviceSmall) {
+ constexpr size_t Size = FallbackPattern.size() * 2;
+ void *Alloc;
+ ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &Alloc));
+
+ ASSERT_SUCCESS(olMemFill(Queue, Alloc, FallbackPattern.size(),
+ FallbackPattern.data(), Size));
+
+ std::vector<unsigned char> HostBuf(Size);
+ ASSERT_SUCCESS(olMemcpy(Queue, HostBuf.data(), Host, Alloc, Device, Size));
+ olSyncQueue(Queue);
+
+ for (size_t I = 0; I < Size; I++)
+ ASSERT_EQ(HostBuf[I], FallbackPattern[I % FallbackPattern.size()]);
+
+ olMemFree(Alloc);
+}
>From 6d4bab40b8e5ef9df5e6a5f806e07cc8fe6ffba9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 16 Jul 2026 08:19:11 +0000
Subject: [PATCH 2/5] [offload] make L0DeviceTy::findCommandQueueGroup return
Expected
---
.../level_zero/include/L0Device.h | 5 ++--
.../level_zero/src/L0Device.cpp | 27 ++++++++++---------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 313bdf1ff7862..d6106e7fa8479 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -24,7 +24,6 @@
#include "PluginInterface.h"
#include <cstdint>
#include <limits>
-#include <optional>
namespace llvm::omp::target::plugin {
@@ -145,9 +144,9 @@ class L0DeviceTy final : public GenericDeviceTy {
DeviceArchTy computeArch() const;
- /// Find the default compute command queue group. Returns std::nullopt if
+ /// Find the default compute command queue group. Returns an Error if
/// the device exposes no compute queue group.
- std::optional<ComputeGroupInfoTy> findCommandQueueGroup();
+ Expected<ComputeGroupInfoTy> findCommandQueueGroup();
/// Helper function to call global constructors or destructors.
Error callGlobalCtorDtorCommon(GenericPluginTy &Plugin, DeviceImageTy &Image,
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index 5130cf147a8be..b90c47f9aa169 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -22,6 +22,7 @@
#include "OffloadAPI.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Object/ELF.h"
+#include "llvm/Support/Error.h"
namespace llvm::omp::target::plugin {
@@ -98,18 +99,18 @@ bool L0DeviceTy::isDeviceIPorNewer(uint32_t Version) const {
return IPVersion.ipVersion >= Version;
}
-/// Find the default compute command queue group. Returns std::nullopt if the
+/// Find the default compute command queue group. Returns an Error if the
/// device exposes no compute queue group.
-std::optional<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
+Expected<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
uint32_t Count = 0;
const auto zeDevice = getZeDevice();
- CALL_ZE_RET(std::nullopt, zeDeviceGetCommandQueueGroupProperties, zeDevice,
- &Count, nullptr);
+ CALL_ZE_RET_ERROR(zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
+ nullptr);
ze_command_queue_group_properties_t Init{
ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES, nullptr, 0, 0, 0};
std::vector<ze_command_queue_group_properties_t> Properties(Count, Init);
- CALL_ZE_RET(std::nullopt, zeDeviceGetCommandQueueGroupProperties, zeDevice,
- &Count, Properties.data());
+ CALL_ZE_RET_ERROR(zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
+ Properties.data());
for (uint32_t I = 0; I < Count; I++) {
// TODO: add a separate set of ordinals for compute queue groups which
// support cooperative kernels.
@@ -120,7 +121,9 @@ std::optional<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
}
}
- return std::nullopt;
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "Device %d (%s) has no compute command queue group",
+ DeviceId, getNameCStr());
}
/// Check if device supports cooperative kernels by checking if any command
@@ -200,12 +203,10 @@ Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
uid += std::to_string(DeviceProperties.uuid.id[n]);
DeviceUuid = std::move(uid);
- auto ComputeGroupInfoOpt = findCommandQueueGroup();
- if (not ComputeGroupInfoOpt)
- return Plugin::error(ErrorCode::UNSUPPORTED,
- "Device %d (%s) has no compute command queue group",
- DeviceId, getNameCStr());
- ComputeGroupInfo = *ComputeGroupInfoOpt;
+ auto ComputeGroupInfoOrErr = findCommandQueueGroup();
+ if (!ComputeGroupInfoOrErr)
+ return ComputeGroupInfoOrErr.takeError();
+ ComputeGroupInfo = *ComputeGroupInfoOrErr;
QueueCache.setCommandMode(getPlugin().getOptions().CommandMode);
SupportsCooperativeKernels = checkCooperativeKernelSupport();
>From 7d8c58ca60091d8dd9dc148121aada5b11cf1f89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 16 Jul 2026 08:44:56 +0000
Subject: [PATCH 3/5] remove redundant allBytesEqual in favor of
StringRef::find_first_not_of
---
.../plugins-nextgen/level_zero/src/L0Queue.cpp | 16 +++-------------
1 file changed, 3 insertions(+), 13 deletions(-)
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index b3d1ddf48b7c2..514a89af4cac3 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -15,6 +15,7 @@
#include "L0Kernel.h"
#include "L0Plugin.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/MathExtras.h"
#include <algorithm>
@@ -62,17 +63,6 @@ Error L0QueueTy::dispatchLaunchKernel(ze_kernel_handle_t Kernel,
KEnv.IsCooperative);
}
-static bool allBytesEqual(const unsigned char *buf, size_t size) {
- if (size == 0)
- return true;
- unsigned char FirstByte = buf[0];
- for (size_t i = 1; i < size; ++i) {
- if (buf[i] != FirstByte)
- return false;
- }
- return true;
-}
-
Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
size_t Size) {
if (Size == 0 || PatternSize == 0)
@@ -84,8 +74,8 @@ Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
return memoryFillImpl(Ptr, Pattern, PatternSize, Size);
}
- const auto *PatternBytes = static_cast<const unsigned char *>(Pattern);
- if (allBytesEqual(PatternBytes, PatternSize)) {
+ llvm::StringRef PatternRef(static_cast<const char*>(Pattern), PatternSize);
+ if (PatternRef.find_first_not_of(PatternRef[0]) == llvm::StringRef::npos) {
// All pattern bytes equal, substutition of 1 as PatternSize is equivalent,
// so native L0 memory fill is still possible.
return memoryFillImpl(Ptr, Pattern, 1, Size);
>From 45d8d0b61ec68a2f396daec3e5d4b0222d26add7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 16 Jul 2026 09:39:19 +0000
Subject: [PATCH 4/5] remove double zeDeviceGetCommandQueueGroupProperties
and create dedicated single struct for the results of the query
---
.../level_zero/include/L0Device.h | 44 +++++++-----
.../level_zero/src/L0Device.cpp | 69 ++++++++-----------
.../level_zero/src/L0Queue.cpp | 2 +-
3 files changed, 57 insertions(+), 58 deletions(-)
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index d6106e7fa8479..dc1ad2ea83661 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -76,7 +76,7 @@ struct L0DeviceIdTy {
: zeId(Device), RootId(RootId), SubId(SubId), CCSId(CCSId) {}
};
-/// Properties of the compute command queue group selected for a device.
+/// Properties of a compute command queue group.
struct ComputeGroupInfoTy {
/// Command queue group ordinal.
uint32_t Ordinal = std::numeric_limits<uint32_t>::max();
@@ -87,6 +87,15 @@ struct ComputeGroupInfoTy {
size_t MaxMemFillPatternSize = 0;
};
+/// Results of scanning a device's command queue groups.
+struct QueueGroupInfoTy {
+ /// The compute command queue group selected as the device default.
+ ComputeGroupInfoTy DefaultCmdQueueGroup;
+ /// Whether any command queue group on this device supports cooperative
+ /// kernels.
+ bool SupportsCooperativeKernels = false;
+};
+
class L0DeviceTy final : public GenericDeviceTy {
// Level Zero Context for this Device.
L0ContextTy &l0Context;
@@ -116,16 +125,13 @@ class L0DeviceTy final : public GenericDeviceTy {
/// L0 Device ID as string.
std::string zeId;
- /// Compute command queue group info for this device. Value is unspecified
- /// unless the device reached a valid initialized state.
- ComputeGroupInfoTy ComputeGroupInfo;
+ /// Command queue group info for this device. Value is unspecified unless the
+ /// device reached a valid initialized state.
+ QueueGroupInfoTy QueueGroupInfo;
/// Command queue index for each device.
uint32_t ComputeIndex = 0;
- /// Whether the device supports cooperative kernels.
- bool SupportsCooperativeKernels = false;
-
/// Lock for this device.
std::mutex Mutex;
@@ -144,17 +150,15 @@ class L0DeviceTy final : public GenericDeviceTy {
DeviceArchTy computeArch() const;
- /// Find the default compute command queue group. Returns an Error if
- /// the device exposes no compute queue group.
- Expected<ComputeGroupInfoTy> findCommandQueueGroup();
+ /// Scan the device's command queue groups, selecting the default compute
+ /// group and detecting cooperative kernel support. Returns an Error if the
+ /// device exposes no compute queue group.
+ Expected<QueueGroupInfoTy> scanQueueGroups();
/// Helper function to call global constructors or destructors.
Error callGlobalCtorDtorCommon(GenericPluginTy &Plugin, DeviceImageTy &Image,
bool IsCtor);
- /// Check if device supports cooperative kernels.
- bool checkCooperativeKernelSupport();
-
public:
L0DeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices,
ze_device_handle_t zeDevice, L0ContextTy &DriverInfo,
@@ -186,7 +190,9 @@ class L0DeviceTy final : public GenericDeviceTy {
Error deinitImpl() override;
ze_device_handle_t getZeDevice() const { return zeDevice; }
- bool supportsCooperativeKernels() const { return SupportsCooperativeKernels; }
+ bool supportsCooperativeKernels() const {
+ return QueueGroupInfo.SupportsCooperativeKernels;
+ }
const L0ContextTy &getL0Context() const { return l0Context; }
L0ContextTy &getL0Context() { return l0Context; }
@@ -350,11 +356,15 @@ class L0DeviceTy final : public GenericDeviceTy {
const std::string_view getUuid() const { return DeviceUuid; }
- uint32_t getComputeEngine() const { return ComputeGroupInfo.Ordinal; }
- uint32_t getNumComputeQueues() const { return ComputeGroupInfo.NumQueues; }
+ uint32_t getComputeEngine() const {
+ return QueueGroupInfo.DefaultCmdQueueGroup.Ordinal;
+ }
+ uint32_t getNumComputeQueues() const {
+ return QueueGroupInfo.DefaultCmdQueueGroup.NumQueues;
+ }
size_t getMaxMemFillPatternSize() {
- return ComputeGroupInfo.MaxMemFillPatternSize;
+ return QueueGroupInfo.DefaultCmdQueueGroup.MaxMemFillPatternSize;
}
void reportDeviceInfo() const;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index b90c47f9aa169..dd967463c37d4 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -99,9 +99,10 @@ bool L0DeviceTy::isDeviceIPorNewer(uint32_t Version) const {
return IPVersion.ipVersion >= Version;
}
-/// Find the default compute command queue group. Returns an Error if the
-/// device exposes no compute queue group.
-Expected<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
+/// Scan the device's command queue groups in a single query, selecting the
+/// default compute group and detecting cooperative kernel support. Returns an
+/// Error if the device exposes no compute queue group.
+Expected<QueueGroupInfoTy> L0DeviceTy::scanQueueGroups() {
uint32_t Count = 0;
const auto zeDevice = getZeDevice();
CALL_ZE_RET_ERROR(zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
@@ -111,41 +112,30 @@ Expected<ComputeGroupInfoTy> L0DeviceTy::findCommandQueueGroup() {
std::vector<ze_command_queue_group_properties_t> Properties(Count, Init);
CALL_ZE_RET_ERROR(zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
Properties.data());
+
+ QueueGroupInfoTy Info;
+ bool FoundComputeGroup = false;
for (uint32_t I = 0; I < Count; I++) {
- // TODO: add a separate set of ordinals for compute queue groups which
- // support cooperative kernels.
- if (Properties[I].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) {
- return ComputeGroupInfoTy{/*Ordinal=*/I,
- /*NumQueues=*/Properties[I].numQueues,
- Properties[I].maxMemoryFillPatternSize};
+ if (!FoundComputeGroup &&
+ (Properties[I].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)) {
+ Info.DefaultCmdQueueGroup =
+ ComputeGroupInfoTy{/*Ordinal=*/I,
+ /*NumQueues=*/Properties[I].numQueues,
+ Properties[I].maxMemoryFillPatternSize};
+ FoundComputeGroup = true;
}
+ // TODO: track exactly which queue groups support cooperative kernels
+ if (Properties[I].flags &
+ ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COOPERATIVE_KERNELS)
+ Info.SupportsCooperativeKernels = true;
}
- return Plugin::error(ErrorCode::UNSUPPORTED,
- "Device %d (%s) has no compute command queue group",
- DeviceId, getNameCStr());
-}
-
-/// Check if device supports cooperative kernels by checking if any command
-/// queue group has the cooperative kernels flag set.
-bool L0DeviceTy::checkCooperativeKernelSupport() {
- uint32_t Count = 0;
- const auto zeDevice = getZeDevice();
- CALL_ZE_RET(false, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
- nullptr);
+ if (!FoundComputeGroup)
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "Device %d (%s) has no compute command queue group",
+ DeviceId, getNameCStr());
- std::vector<ze_command_queue_group_properties_t> Properties(
- Count,
- {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES, nullptr, 0, 0, 0});
- CALL_ZE_RET(false, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
- Properties.data());
-
- for (auto &Property : Properties)
- if (Property.flags &
- ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COOPERATIVE_KERNELS)
- return true;
-
- return false;
+ return Info;
}
void L0DeviceTy::reportDeviceInfo() const {
@@ -203,14 +193,12 @@ Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
uid += std::to_string(DeviceProperties.uuid.id[n]);
DeviceUuid = std::move(uid);
- auto ComputeGroupInfoOrErr = findCommandQueueGroup();
- if (!ComputeGroupInfoOrErr)
- return ComputeGroupInfoOrErr.takeError();
- ComputeGroupInfo = *ComputeGroupInfoOrErr;
+ auto QueueGroupInfoOrErr = scanQueueGroups();
+ if (!QueueGroupInfoOrErr)
+ return QueueGroupInfoOrErr.takeError();
+ QueueGroupInfo = *QueueGroupInfoOrErr;
QueueCache.setCommandMode(getPlugin().getOptions().CommandMode);
- SupportsCooperativeKernels = checkCooperativeKernelSupport();
-
if (auto Err = MemAllocator.initDevicePools(*this, Options))
return Err;
l0Context.getHostMemAllocator().updateMaxAllocSize(*this);
@@ -562,7 +550,8 @@ Expected<InfoTreeNode> L0DeviceTy::obtainInfoImpl() {
Info.add("Single FP Capabilities", SingleFPCapabilities, "",
DeviceInfo::SINGLE_FP_CONFIG);
- Info.add("Cooperative launch support", SupportsCooperativeKernels, "",
+ Info.add("Cooperative launch support",
+ QueueGroupInfo.SupportsCooperativeKernels, "",
DeviceInfo::COOPERATIVE_LAUNCH_SUPPORT);
return Info;
}
diff --git a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
index 514a89af4cac3..fca6b81084345 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Queue.cpp
@@ -74,7 +74,7 @@ Error L0QueueTy::memoryFill(void *Ptr, const void *Pattern, size_t PatternSize,
return memoryFillImpl(Ptr, Pattern, PatternSize, Size);
}
- llvm::StringRef PatternRef(static_cast<const char*>(Pattern), PatternSize);
+ llvm::StringRef PatternRef(static_cast<const char *>(Pattern), PatternSize);
if (PatternRef.find_first_not_of(PatternRef[0]) == llvm::StringRef::npos) {
// All pattern bytes equal, substutition of 1 as PatternSize is equivalent,
// so native L0 memory fill is still possible.
>From 87b3651664d10639a5ff53d982a6a52153214551 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Trusi=C5=82=C5=82o?= <jan.trusillo at intel.com>
Date: Thu, 16 Jul 2026 10:45:47 +0000
Subject: [PATCH 5/5] change confusing queue group info struct naming
---
.../level_zero/include/L0Device.h | 18 +++++++++---------
.../level_zero/src/L0Device.cpp | 10 +++++-----
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index dc1ad2ea83661..e5b7772b58daf 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -77,7 +77,7 @@ struct L0DeviceIdTy {
};
/// Properties of a compute command queue group.
-struct ComputeGroupInfoTy {
+struct ComputeQueueGroupTy {
/// Command queue group ordinal.
uint32_t Ordinal = std::numeric_limits<uint32_t>::max();
/// Number of queues in the group.
@@ -88,9 +88,9 @@ struct ComputeGroupInfoTy {
};
/// Results of scanning a device's command queue groups.
-struct QueueGroupInfoTy {
+struct DeviceQueueConfigTy {
/// The compute command queue group selected as the device default.
- ComputeGroupInfoTy DefaultCmdQueueGroup;
+ ComputeQueueGroupTy DefaultCmdQueueGroup;
/// Whether any command queue group on this device supports cooperative
/// kernels.
bool SupportsCooperativeKernels = false;
@@ -127,7 +127,7 @@ class L0DeviceTy final : public GenericDeviceTy {
/// Command queue group info for this device. Value is unspecified unless the
/// device reached a valid initialized state.
- QueueGroupInfoTy QueueGroupInfo;
+ DeviceQueueConfigTy QueueConfig;
/// Command queue index for each device.
uint32_t ComputeIndex = 0;
@@ -153,7 +153,7 @@ class L0DeviceTy final : public GenericDeviceTy {
/// Scan the device's command queue groups, selecting the default compute
/// group and detecting cooperative kernel support. Returns an Error if the
/// device exposes no compute queue group.
- Expected<QueueGroupInfoTy> scanQueueGroups();
+ Expected<DeviceQueueConfigTy> scanQueueGroups();
/// Helper function to call global constructors or destructors.
Error callGlobalCtorDtorCommon(GenericPluginTy &Plugin, DeviceImageTy &Image,
@@ -191,7 +191,7 @@ class L0DeviceTy final : public GenericDeviceTy {
ze_device_handle_t getZeDevice() const { return zeDevice; }
bool supportsCooperativeKernels() const {
- return QueueGroupInfo.SupportsCooperativeKernels;
+ return QueueConfig.SupportsCooperativeKernels;
}
const L0ContextTy &getL0Context() const { return l0Context; }
@@ -357,14 +357,14 @@ class L0DeviceTy final : public GenericDeviceTy {
const std::string_view getUuid() const { return DeviceUuid; }
uint32_t getComputeEngine() const {
- return QueueGroupInfo.DefaultCmdQueueGroup.Ordinal;
+ return QueueConfig.DefaultCmdQueueGroup.Ordinal;
}
uint32_t getNumComputeQueues() const {
- return QueueGroupInfo.DefaultCmdQueueGroup.NumQueues;
+ return QueueConfig.DefaultCmdQueueGroup.NumQueues;
}
size_t getMaxMemFillPatternSize() {
- return QueueGroupInfo.DefaultCmdQueueGroup.MaxMemFillPatternSize;
+ return QueueConfig.DefaultCmdQueueGroup.MaxMemFillPatternSize;
}
void reportDeviceInfo() const;
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index dd967463c37d4..852d0b5eeeb59 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -102,7 +102,7 @@ bool L0DeviceTy::isDeviceIPorNewer(uint32_t Version) const {
/// Scan the device's command queue groups in a single query, selecting the
/// default compute group and detecting cooperative kernel support. Returns an
/// Error if the device exposes no compute queue group.
-Expected<QueueGroupInfoTy> L0DeviceTy::scanQueueGroups() {
+Expected<DeviceQueueConfigTy> L0DeviceTy::scanQueueGroups() {
uint32_t Count = 0;
const auto zeDevice = getZeDevice();
CALL_ZE_RET_ERROR(zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
@@ -113,13 +113,13 @@ Expected<QueueGroupInfoTy> L0DeviceTy::scanQueueGroups() {
CALL_ZE_RET_ERROR(zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
Properties.data());
- QueueGroupInfoTy Info;
+ DeviceQueueConfigTy Info;
bool FoundComputeGroup = false;
for (uint32_t I = 0; I < Count; I++) {
if (!FoundComputeGroup &&
(Properties[I].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)) {
Info.DefaultCmdQueueGroup =
- ComputeGroupInfoTy{/*Ordinal=*/I,
+ ComputeQueueGroupTy{/*Ordinal=*/I,
/*NumQueues=*/Properties[I].numQueues,
Properties[I].maxMemoryFillPatternSize};
FoundComputeGroup = true;
@@ -196,7 +196,7 @@ Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
auto QueueGroupInfoOrErr = scanQueueGroups();
if (!QueueGroupInfoOrErr)
return QueueGroupInfoOrErr.takeError();
- QueueGroupInfo = *QueueGroupInfoOrErr;
+ QueueConfig = *QueueGroupInfoOrErr;
QueueCache.setCommandMode(getPlugin().getOptions().CommandMode);
if (auto Err = MemAllocator.initDevicePools(*this, Options))
@@ -551,7 +551,7 @@ Expected<InfoTreeNode> L0DeviceTy::obtainInfoImpl() {
DeviceInfo::SINGLE_FP_CONFIG);
Info.add("Cooperative launch support",
- QueueGroupInfo.SupportsCooperativeKernels, "",
+ QueueConfig.SupportsCooperativeKernels, "",
DeviceInfo::COOPERATIVE_LAUNCH_SUPPORT);
return Info;
}
More information about the llvm-commits
mailing list