[llvm] [OFFLOAD][OMP] Move OpenMP kernel argument processing from plugins (PR #213867)
Alex Duran via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 03:42:50 PDT 2026
https://github.com/adurang updated https://github.com/llvm/llvm-project/pull/213867
>From 700132ebc81e672ccd8d38378fbe7712005f8b9a Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Tue, 4 Aug 2026 00:56:37 -0700
Subject: [PATCH 1/6] [OFFLOAD][OMP] Move OpenMP specific kernel argument
processing out of the plugins
This PR moves part of the OpenMP specific code that is inside the common code of the
plugins that converts the kernel arguments from the OpenMP ABI to the expected
format by the plugins. Also it handles the additional argument for the KernelLaunch
environment.
It also decouples the plugin interface structs from the OpenMP specific ABI (KernelArgsTy)
so changes to this interface do not require changes to the OpenMP ABI anymore. This also
allows to merge what was KernelArgsTy and LaunchParamsTy into a single struct with all
the information. Because of this there's a number of small changes scattered through
the plugin infrastructure.
There is still some more OpenMP specific code that should be moved out eventually (and
because of this we had to retain some of that informationt in the new
KernelLaunchArgsTy for now) but I didn't want to complicate this PR further.
---
offload/liboffload/src/OffloadImpl.cpp | 13 +-
offload/libomptarget/device.cpp | 72 ++++++++-
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 38 +++--
.../common/include/PluginInterface.h | 83 +++++++---
.../common/include/RecordReplay.h | 28 ++--
.../common/src/PluginInterface.cpp | 144 ++++++------------
.../common/src/RecordReplay.cpp | 28 ++--
offload/plugins-nextgen/cuda/src/rtl.cpp | 17 +--
offload/plugins-nextgen/host/src/rtl.cpp | 8 +-
.../level_zero/include/L0Kernel.h | 10 +-
.../level_zero/src/L0Device.cpp | 4 +-
.../level_zero/src/L0Kernel.cpp | 12 +-
12 files changed, 250 insertions(+), 207 deletions(-)
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 63fe726fbe544..277df26fbce8a 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -1239,8 +1239,11 @@ Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,
"provided symbol is not a kernel");
auto *QueueImpl = Queue ? Queue->AsyncInfo : nullptr;
- KernelArgsTy LaunchArgs{};
+ KernelLaunchArgsTy LaunchArgs{};
LaunchArgs.NumArgs = static_cast<uint32_t>(NumArgs);
+ LaunchArgs.Args = ArgPtrs;
+ LaunchArgs.ArgSizes =
+ reinterpret_cast<int64_t *>(const_cast<size_t *>(ArgSizes));
LaunchArgs.UserNumBlocks[0] = LaunchSizeArgs->NumGroups.x;
LaunchArgs.UserNumBlocks[1] = LaunchSizeArgs->NumGroups.y;
LaunchArgs.UserNumBlocks[2] = LaunchSizeArgs->NumGroups.z;
@@ -1265,14 +1268,10 @@ Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,
}
AsyncInfoWrapperTy AsyncInfoWrapper(*DeviceImpl, QueueImpl);
- LaunchArgs.ArgPtrs = ArgPtrs;
- LaunchArgs.ArgSizes =
- reinterpret_cast<int64_t *>(const_cast<size_t *>(ArgSizes));
- LaunchArgs.Flags.IsPtrArgs = true;
auto *KernelImpl = std::get<GenericKernelTy *>(Kernel->PluginImpl);
- auto Err = KernelImpl->launch(*DeviceImpl, LaunchArgs.ArgPtrs, nullptr,
- LaunchArgs, nullptr, AsyncInfoWrapper);
+ auto Err =
+ KernelImpl->launch(*DeviceImpl, LaunchArgs, nullptr, AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
if (Err)
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 18339e9afe975..8bdde45dfc494 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -23,8 +23,10 @@
#include "rtl.h"
#include "Shared/EnvironmentVar.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/Support/Error.h"
+#include <algorithm>
#include <cassert>
#include <climits>
#include <cstdint>
@@ -354,13 +356,79 @@ int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
return OFFLOAD_SUCCESS;
}
+/// Resolve \p NumArgs (base pointer, offset) pairs into a flattened array of
+/// argument-value pointers suitable for a kernel launch, writing the result
+/// into \p LaunchArgs.NumArgs/Args.
+static void resolveKernelLaunchParams(void **TgtArgs, ptrdiff_t *TgtOffsets,
+ uint32_t NumArgs,
+ llvm::SmallVector<void *> &Args,
+ llvm::SmallVector<void *> &Ptrs,
+ KernelLaunchArgsTy &LaunchArgs) {
+ LaunchArgs.NumArgs = NumArgs;
+ if (NumArgs == 0)
+ return;
+
+ Args.resize(NumArgs);
+ Ptrs.resize(NumArgs);
+
+ for (uint32_t I = 0; I < NumArgs; ++I)
+ Args[I] = reinterpret_cast<void *>(reinterpret_cast<intptr_t>(TgtArgs[I]) +
+ TgtOffsets[I]);
+ for (uint32_t I = 0; I < NumArgs; ++I)
+ Ptrs[I] = &Args[I];
+
+ LaunchArgs.Args = &Ptrs[0];
+}
+
// Run region on device
int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs,
KernelExtraArgsTy *KernelExtraArgs,
AsyncInfoTy &AsyncInfo) {
- return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets,
- &KernelArgs, KernelExtraArgs, AsyncInfo);
+ llvm::SmallVector<void *> Args, Ptrs;
+
+ KernelLaunchArgsTy LaunchArgs;
+ LaunchArgs.OmpABIVersion = KernelArgs.Version;
+ LaunchArgs.ArgSizes = KernelArgs.ArgSizes;
+ LaunchArgs.Tripcount = KernelArgs.Tripcount;
+ LaunchArgs.DynCGroupMem = KernelArgs.DynCGroupMem;
+ std::copy(std::begin(KernelArgs.UserNumBlocks),
+ std::end(KernelArgs.UserNumBlocks), LaunchArgs.UserNumBlocks);
+ std::copy(std::begin(KernelArgs.UserThreadLimit),
+ std::end(KernelArgs.UserThreadLimit), LaunchArgs.UserThreadLimit);
+ LaunchArgs.Flags.Cooperative = KernelArgs.Flags.Cooperative;
+ LaunchArgs.Flags.StrictBlocksAndThreads =
+ KernelArgs.Flags.StrictBlocksAndThreads;
+ LaunchArgs.Flags.DynCGroupMemFallback = KernelArgs.Flags.DynCGroupMemFallback;
+
+ if (KernelArgs.Flags.IsCUDA) {
+ // Kernel languages (CUDA/HIP) pass an already-flattened argument-pointer
+ // array through KernelArgs.ArgPtrs instead of using the OpenMP
+ // base-pointer/offset argument scheme.
+ auto *LaunchParams =
+ reinterpret_cast<KernelLaunchParamsTy *>(KernelArgs.ArgPtrs);
+ LaunchArgs.NumArgs = LaunchParams->NumArgs;
+ LaunchArgs.Args = LaunchParams->Args;
+ } else {
+ resolveKernelLaunchParams(TgtVarsPtr, TgtOffsets, KernelArgs.NumArgs, Args,
+ Ptrs, LaunchArgs);
+ // The dyn_ptr slot is reserved by the host (version >= 4) or by
+ // upgradeKernelArgs (version 3) as the last element of the argument
+ // array. Version 3 device kernels expect it first instead, so rotate it
+ // to the front to match that ABI.
+ if (KernelArgs.NumArgs > 0 &&
+ KernelArgs.Version >= OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) {
+ if (KernelArgs.Version == OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) {
+ std::rotate(Args.begin(), Args.end() - 1, Args.end());
+ LaunchArgs.DynPtrSlot = &Args[0];
+ } else {
+ LaunchArgs.DynPtrSlot = &Args[KernelArgs.NumArgs - 1];
+ }
+ }
+ }
+
+ return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, LaunchArgs,
+ KernelExtraArgs, AsyncInfo);
}
// Run region on device
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 65ef83c394f6a..53bd150732887 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -648,7 +648,7 @@ struct AMDGPUKernelTy : public GenericKernelTy {
/// Launch the AMDGPU kernel function.
Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
uint32_t NumBlocks[3], uint32_t DynBlockMemSize,
- KernelArgsTy &KernelArgs, KernelLaunchParamsTy LaunchParams,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
/// Return maximum block size for maximum occupancy
@@ -663,7 +663,8 @@ struct AMDGPUKernelTy : public GenericKernelTy {
/// Print more elaborate kernel launch info for AMDGPU
Error printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs, uint32_t NumThreads[3],
+ const KernelLaunchArgsTy &LaunchArgs,
+ uint32_t NumThreads[3],
uint32_t NumBlocks[3]) const override;
/// Get group and private segment kernel size.
@@ -3561,11 +3562,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
AsyncInfoWrapperTy AsyncInfoWrapper(*this, nullptr);
- KernelArgsTy KernelArgs = {};
+ KernelLaunchArgsTy LaunchArgs = {};
uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
- auto Err = AMDGPUKernel.launchImpl(
- *this, NumBlocksAndThreads, NumBlocksAndThreads, 0, KernelArgs,
- KernelLaunchParamsTy{}, AsyncInfoWrapper);
+ auto Err =
+ AMDGPUKernel.launchImpl(*this, NumBlocksAndThreads, NumBlocksAndThreads,
+ 0, LaunchArgs, AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
return Err;
@@ -4227,11 +4228,10 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
uint32_t DynBlockMemSize,
- KernelArgsTy &KernelArgs,
- KernelLaunchParamsTy LaunchParams,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const {
// Cooperative kernel launch is not yet supported for AMDGPU
- if (KernelArgs.Flags.Cooperative)
+ if (LaunchArgs.Flags.Cooperative)
return Plugin::error(ErrorCode::UNSUPPORTED,
"cooperative kernel launch not supported for AMDGPU");
@@ -4250,9 +4250,9 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
// Copy explicit arguments.
size_t ExplicitEnd = 0;
- if (LaunchParams.Args) {
+ if (LaunchArgs.Args) {
const auto &ArgMDs = KernelInfo.ArgMDs;
- uint32_t NumArgs = LaunchParams.NumArgs;
+ uint32_t NumArgs = LaunchArgs.NumArgs;
if (NumArgs > ArgMDs.size())
return Plugin::error(
@@ -4263,8 +4263,7 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
for (size_t I = 0; I < NumArgs; I++) {
auto [Offset, Size] = ArgMDs[I];
- std::memcpy(utils::advancePtr(AllArgs, Offset), LaunchParams.Args[I],
- Size);
+ std::memcpy(utils::advancePtr(AllArgs, Offset), LaunchArgs.Args[I], Size);
}
auto [Offset, Size] = ArgMDs[NumArgs - 1];
@@ -4309,7 +4308,7 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
: 1 + (NumBlocks[1] * NumThreads[1] != 1));
hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::DynamicLdsSize, ImplArgsSize,
- KernelArgs.DynCGroupMem);
+ LaunchArgs.DynCGroupMem);
}
// HSA requires the group segment size to include both static and dynamic.
@@ -4321,10 +4320,9 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
ArgsMemoryManager);
}
-Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs,
- uint32_t NumThreads[3],
- uint32_t NumBlocks[3]) const {
+Error AMDGPUKernelTy::printLaunchInfoDetails(
+ GenericDeviceTy &GenericDevice, const KernelLaunchArgsTy &LaunchArgs,
+ uint32_t NumThreads[3], uint32_t NumBlocks[3]) const {
// Only do all this when the output is requested
if (!(getInfoLevel() & OMP_INFOTYPE_PLUGIN_KERNEL))
return Plugin::success();
@@ -4334,8 +4332,8 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
auto *ThreadsPerGroup = NumThreads;
// Kernel Arguments Info
- auto ArgNum = KernelArgs.NumArgs;
- auto LoopTripCount = KernelArgs.Tripcount;
+ auto ArgNum = LaunchArgs.NumArgs;
+ auto LoopTripCount = LaunchArgs.Tripcount;
// Details for AMDGPU kernels (read from image)
// https://www.llvm.org/docs/AMDGPUUsage.html#code-object-v4-metadata
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 67ebfbc943fdc..a669cddd0b76e 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -424,6 +424,45 @@ class DeviceImageTy {
}
};
+/// The subset of KernelArgsTy fields the plugin interface needs to launch a
+/// kernel, plus the resolved argument-pointer array. Unlike KernelArgsTy,
+/// this struct is populated by libomptarget on the stack for every launch,
+/// so it is never aliased onto compiler-emitted memory and may be extended
+/// freely.
+struct KernelLaunchArgsTy {
+ /// Version of KernelArgsTy this launch was built from, for ABI
+ /// compatibility checks.
+ uint32_t OmpABIVersion = 0;
+ /// Number of kernel arguments in \p Args.
+ uint32_t NumArgs = 0;
+ /// Array of \p NumArgs pointers, each pointing at one argument's value,
+ /// with any offsets already resolved.
+ void **Args = nullptr;
+ /// Size of the argument data in bytes, one entry per \p Args element,
+ /// possibly null.
+ int64_t *ArgSizes = nullptr;
+ /// Address of the element of \p Args reserved for the kernel launch
+ /// environment (dyn_ptr), or null if this launch has no such slot. The
+ /// caller owns the storage it points into; the plugin fills it in once it
+ /// has computed the actual (device-side) value.
+ void **DynPtrSlot = nullptr;
+ /// Tripcount for the teams / distribute loop, 0 otherwise.
+ uint64_t Tripcount = 0;
+ /// Amount of dynamic cgroup memory requested.
+ uint32_t DynCGroupMem = 0;
+ /// User-requested number of blocks (for x,y,z dimension).
+ uint32_t UserNumBlocks[3] = {0, 0, 0};
+ /// User-requested number of threads (for x,y,z dimension).
+ uint32_t UserThreadLimit[3] = {0, 0, 0};
+ struct {
+ uint64_t Cooperative : 1; // Was this kernel spawned as cooperative.
+ uint64_t StrictBlocksAndThreads
+ : 1; // The user-requested number of blocks and threads are strict.
+ uint64_t DynCGroupMemFallback : 2; // The fallback for dynamic cgroup mem.
+ uint64_t Unused : 60;
+ } Flags = {0, 0, 0, 0};
+};
+
/// Class implementing common functionalities of offload kernels. Each plugin
/// should define the specific kernel class, derive from this generic one, and
/// implement the necessary virtual function members.
@@ -440,15 +479,18 @@ struct GenericKernelTy {
DeviceImageTy &Image) = 0;
/// Launch the kernel on the specific device. The device must be the same
- /// one used to initialize the kernel.
- Error launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
- ptrdiff_t *ArgOffsets, KernelArgsTy &KernelArgs,
+ /// one used to initialize the kernel. \p LaunchArgs.Args is the flattened
+ /// argument-pointer array to pass to the kernel, with any offsets already
+ /// resolved. \p LaunchArgs.DynPtrSlot, if non-null, points at the element
+ /// of it reserved for the kernel launch environment (dyn_ptr); the caller
+ /// owns the storage it points into.
+ Error launch(GenericDeviceTy &GenericDevice, KernelLaunchArgsTy &LaunchArgs,
KernelExtraArgsTy *KernelExtraArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const;
virtual Error launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
- uint32_t DynBlockMemSize, KernelArgsTy &KernelArgs,
- KernelLaunchParamsTy LaunchParams,
+ uint32_t DynBlockMemSize,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const = 0;
virtual Expected<uint64_t> maxGroupSize(GenericDeviceTy &GenericDevice,
@@ -488,7 +530,7 @@ struct GenericKernelTy {
/// \p NumBlocks0 is the number of blocks for this launch and is used to size
/// the reduction buffer.
Expected<KernelLaunchEnvironmentTy *> getKernelLaunchEnvironment(
- GenericDeviceTy &GenericDevice, const KernelArgsTy &KernelArgs,
+ GenericDeviceTy &GenericDevice, const KernelLaunchArgsTy &LaunchArgs,
const DynBlockMemConfTy &DynBlockMemConf,
AsyncInfoWrapperTy &AsyncInfoWrapper, uint32_t NumBlocks0) const;
@@ -525,31 +567,23 @@ struct GenericKernelTy {
/// Prints generic kernel launch information.
Error printLaunchInfo(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs, uint32_t NumThreads[3],
- uint32_t NumBlocks[3]) const;
+ const KernelLaunchArgsTy &LaunchArgs,
+ uint32_t NumThreads[3], uint32_t NumBlocks[3]) const;
/// Prints plugin-specific kernel launch information after generic kernel
/// launch information
virtual Error printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs,
+ const KernelLaunchArgsTy &LaunchArgs,
uint32_t NumThreads[3],
uint32_t NumBlocks[3]) const;
private:
/// Prepare the block memory buffer requested for the kernel and execute the
/// specified fallback if necessary.
- Expected<DynBlockMemConfTy> prepareBlockMemory(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs,
- uint32_t NumBlocks) const;
-
- /// Prepare the arguments before launching the kernel.
- KernelLaunchParamsTy
- prepareArgs(GenericDeviceTy &GenericDevice, void **ArgPtrs,
- ptrdiff_t *ArgOffsets, uint32_t &NumArgs,
- llvm::SmallVectorImpl<void *> &Args,
- llvm::SmallVectorImpl<void *> &Ptrs,
- KernelLaunchEnvironmentTy *KernelLaunchEnvironment,
- uint32_t Version) const;
+ Expected<DynBlockMemConfTy>
+ prepareBlockMemory(GenericDeviceTy &GenericDevice,
+ const KernelLaunchArgsTy &LaunchArgs,
+ uint32_t NumBlocks) const;
/// Get the effective number of threads for the kernel based on the
/// user-defined number of threads.
@@ -1099,8 +1133,7 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
}
/// Run the kernel associated with \p EntryPtr
- Error launchKernel(void *EntryPtr, void **ArgPtrs, ptrdiff_t *ArgOffsets,
- KernelArgsTy &KernelArgs,
+ Error launchKernel(void *EntryPtr, KernelLaunchArgsTy &LaunchArgs,
KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfo);
@@ -1714,8 +1747,8 @@ struct GenericPluginTy {
int32_t data_fence(int32_t DeviceId, __tgt_async_info *AsyncInfo);
/// Begin executing a kernel on the given device.
- int32_t launch_kernel(int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs,
- ptrdiff_t *TgtOffsets, KernelArgsTy *KernelArgs,
+ int32_t launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
+ KernelLaunchArgsTy &LaunchArgs,
KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfoPtr);
diff --git a/offload/plugins-nextgen/common/include/RecordReplay.h b/offload/plugins-nextgen/common/include/RecordReplay.h
index 4c81f64620f32..5402bb64d8b24 100644
--- a/offload/plugins-nextgen/common/include/RecordReplay.h
+++ b/offload/plugins-nextgen/common/include/RecordReplay.h
@@ -39,6 +39,7 @@ namespace plugin {
struct GenericKernelTy;
struct GenericDeviceTy;
+struct KernelLaunchArgsTy;
struct RecordReplayTy {
protected:
@@ -204,11 +205,12 @@ struct RecordReplayTy {
/// executing the kernel. This phase can include the recording of memory
/// snapshot, the record descriptor and the globals. When replaying, only the
/// instance is registered.
- Expected<HandleTy>
- recordPrologue(const GenericKernelTy &Kernel, const KernelArgsTy &KernelArgs,
- const KernelExtraArgsTy *KernelExtraArgs,
- const KernelLaunchParamsTy &LaunchParams, uint32_t NumTeams[3],
- uint32_t NumThreads[3], uint32_t SharedMemorySize);
+ Expected<HandleTy> recordPrologue(const GenericKernelTy &Kernel,
+ const KernelLaunchArgsTy &LaunchArgs,
+ const KernelExtraArgsTy *KernelExtraArgs,
+ uint32_t NumTeams[3],
+ uint32_t NumThreads[3],
+ uint32_t SharedMemorySize);
/// Record the epilogue if necessary, which can include the memory snapshot
/// when recording or replaying.
@@ -242,10 +244,9 @@ struct RecordReplayTy {
KernelReplayOutcomeTy &Outcome);
/// Record the prologue data.
- virtual Error
- recordPrologueImpl(const GenericKernelTy &Kernel, const InstanceTy &Instance,
- const KernelArgsTy &KernelArgs,
- const KernelLaunchParamsTy &LaunchParams) = 0;
+ virtual Error recordPrologueImpl(const GenericKernelTy &Kernel,
+ const InstanceTy &Instance,
+ const KernelLaunchArgsTy &LaunchArgs) = 0;
/// Record the epilogue data.
virtual Error recordEpilogueImpl(const GenericKernelTy &Kernel,
@@ -254,8 +255,7 @@ struct RecordReplayTy {
/// Record the descriptor of the kernel.
virtual Error recordDescImpl(const GenericKernelTy &Kernel,
const InstanceTy &Instance,
- const KernelArgsTy &KernelArgs,
- const KernelLaunchParamsTy &LaunchParams) = 0;
+ const KernelLaunchArgsTy &LaunchArgs) = 0;
/// Get a string with the filename.
virtual SmallString<128> getFilenameImpl(const InstanceTy &Instance,
@@ -274,14 +274,12 @@ struct NativeRecordReplayTy : public RecordReplayTy {
private:
Error recordPrologueImpl(const GenericKernelTy &Kernel,
const InstanceTy &Instance,
- const KernelArgsTy &KernelArgs,
- const KernelLaunchParamsTy &LaunchParams) override;
+ const KernelLaunchArgsTy &LaunchArgs) override;
Error recordEpilogueImpl(const GenericKernelTy &Kernel,
const InstanceTy &Instance) override;
Error recordDescImpl(const GenericKernelTy &Kernel,
const InstanceTy &Instance,
- const KernelArgsTy &KernelArgs,
- const KernelLaunchParamsTy &LaunchParams) override;
+ const KernelLaunchArgsTy &LaunchArgs) override;
/// Get a string with the filename.
SmallString<128> getFilenameImpl(const InstanceTy &Instance, FileTy FileType,
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 7b821e77df179..e283e6c97ec72 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -108,26 +108,26 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
Expected<KernelLaunchEnvironmentTy *>
GenericKernelTy::getKernelLaunchEnvironment(
- GenericDeviceTy &GenericDevice, const KernelArgsTy &KernelArgs,
+ GenericDeviceTy &GenericDevice, const KernelLaunchArgsTy &LaunchArgs,
const DynBlockMemConfTy &DynBlockMemConf,
AsyncInfoWrapperTy &AsyncInfoWrapper, uint32_t NumBlocks0) const {
// Ctor/Dtor have no arguments, replaying uses the original kernel launch
- // environment. Older versions of the compiler do not generate a kernel
- // launch environment.
+ // environment, and launches with no reserved dyn_ptr slot (e.g. older
+ // compiler versions, or non-OpenMP launches) have nowhere to store one.
if ((GenericDevice.getRecordReplay() &&
GenericDevice.getRecordReplay()->isReplaying()) ||
- KernelArgs.Version < OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR)
+ !LaunchArgs.DynPtrSlot)
return nullptr;
const auto &RedCfg = KernelEnvironment.Configuration;
const bool NeedsReductionBuffer = RedCfg.ReductionDataSize != 0;
- if (NeedsReductionBuffer && KernelArgs.Version < OMP_KERNEL_ARG_VERSION)
+ if (NeedsReductionBuffer && LaunchArgs.OmpABIVersion < OMP_KERNEL_ARG_VERSION)
return Plugin::error(ErrorCode::INVALID_BINARY,
"kernel was built against an older OpenMP "
"kernel-launch-environment ABI (v%u); current "
"runtime requires v%u for cross-team reductions",
- KernelArgs.Version, OMP_KERNEL_ARG_VERSION);
- if (!NeedsReductionBuffer && !KernelArgs.DynCGroupMem)
+ LaunchArgs.OmpABIVersion, OMP_KERNEL_ARG_VERSION);
+ if (!NeedsReductionBuffer && !LaunchArgs.DynCGroupMem)
return reinterpret_cast<KernelLaunchEnvironmentTy *>(~0);
auto AllocOrErr = GenericDevice.dataAlloc(
@@ -177,7 +177,7 @@ GenericKernelTy::getKernelLaunchEnvironment(
}
Error GenericKernelTy::printLaunchInfo(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs,
+ const KernelLaunchArgsTy &LaunchArgs,
uint32_t NumThreads[3],
uint32_t NumBlocks[3]) const {
INFO(OMP_INFOTYPE_PLUGIN_KERNEL, GenericDevice.getDeviceId(),
@@ -185,23 +185,22 @@ Error GenericKernelTy::printLaunchInfo(GenericDeviceTy &GenericDevice,
"%s mode\n",
getName(), NumBlocks[0], NumBlocks[1], NumBlocks[2], NumThreads[0],
NumThreads[1], NumThreads[2], getExecutionModeName());
- return printLaunchInfoDetails(GenericDevice, KernelArgs, NumThreads,
+ return printLaunchInfoDetails(GenericDevice, LaunchArgs, NumThreads,
NumBlocks);
}
-Error GenericKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs,
- uint32_t NumThreads[3],
- uint32_t NumBlocks[3]) const {
+Error GenericKernelTy::printLaunchInfoDetails(
+ GenericDeviceTy &GenericDevice, const KernelLaunchArgsTy &LaunchArgs,
+ uint32_t NumThreads[3], uint32_t NumBlocks[3]) const {
return Plugin::success();
}
Expected<DynBlockMemConfTy>
GenericKernelTy::prepareBlockMemory(GenericDeviceTy &GenericDevice,
- KernelArgsTy &KernelArgs,
+ const KernelLaunchArgsTy &LaunchArgs,
uint32_t NumBlocks) const {
uint32_t MaxBlockMemSize = GenericDevice.getMaxBlockSharedMemSize();
- uint32_t DynBlockMemSize = KernelArgs.DynCGroupMem;
+ uint32_t DynBlockMemSize = LaunchArgs.DynCGroupMem;
uint32_t TotalBlockMemSize = StaticBlockMemSize + DynBlockMemSize;
uint32_t DynNativeBlockMemSize = DynBlockMemSize;
void *DynFallbackPtr = nullptr;
@@ -212,7 +211,7 @@ GenericKernelTy::prepareBlockMemory(GenericDeviceTy &GenericDevice,
"Static block memory size exceeds maximum");
// No enough block memory to cover dynamic one, and the fallback is aborting.
if (static_cast<DynCGroupMemFallbackType>(
- KernelArgs.Flags.DynCGroupMemFallback) ==
+ LaunchArgs.Flags.DynCGroupMemFallback) ==
DynCGroupMemFallbackType::Abort &&
TotalBlockMemSize > MaxBlockMemSize)
return Plugin::error(
@@ -224,7 +223,7 @@ GenericKernelTy::prepareBlockMemory(GenericDeviceTy &GenericDevice,
// Launch without native dynamic block memory.
DynNativeBlockMemSize = 0;
DynFallback = static_cast<DynCGroupMemFallbackType>(
- KernelArgs.Flags.DynCGroupMemFallback);
+ LaunchArgs.Flags.DynCGroupMemFallback);
if (DynFallback != DynCGroupMemFallbackType::DefaultMem) {
// Do not provide any memory as fallback.
DynBlockMemSize = 0;
@@ -243,19 +242,16 @@ GenericKernelTy::prepareBlockMemory(GenericDeviceTy &GenericDevice,
DynFallbackPtr};
}
-Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
- ptrdiff_t *ArgOffsets, KernelArgsTy &KernelArgs,
+Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice,
+ KernelLaunchArgsTy &LaunchArgs,
KernelExtraArgsTy *KernelExtraArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const {
- llvm::SmallVector<void *, 16> Args;
- llvm::SmallVector<void *, 16> Ptrs;
-
- uint32_t EffectiveNumThreads[3] = {KernelArgs.UserThreadLimit[0],
- KernelArgs.UserThreadLimit[1],
- KernelArgs.UserThreadLimit[2]};
- uint32_t EffectiveNumBlocks[3] = {KernelArgs.UserNumBlocks[0],
- KernelArgs.UserNumBlocks[1],
- KernelArgs.UserNumBlocks[2]};
+ uint32_t EffectiveNumThreads[3] = {LaunchArgs.UserThreadLimit[0],
+ LaunchArgs.UserThreadLimit[1],
+ LaunchArgs.UserThreadLimit[2]};
+ uint32_t EffectiveNumBlocks[3] = {LaunchArgs.UserNumBlocks[0],
+ LaunchArgs.UserNumBlocks[1],
+ LaunchArgs.UserNumBlocks[2]};
// Multidimensional is only supported with bare mode for now.
assert(isBareMode() ||
@@ -264,24 +260,24 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
"Non-bare mode should only use the first thread and block "
"dimensions");
- assert(!KernelArgs.Flags.StrictBlocksAndThreads ||
+ assert(!LaunchArgs.Flags.StrictBlocksAndThreads ||
EffectiveNumThreads[0] > 0 && EffectiveNumThreads[1] > 0 &&
EffectiveNumThreads[2] > 0 && EffectiveNumBlocks[0] > 0 &&
EffectiveNumBlocks[1] > 0 && EffectiveNumBlocks[2] > 0 &&
"Strict requires number of blocks and threads greater than zero");
// Calculate or adjust the effective number of threads and blocks if needed.
- if (!KernelArgs.Flags.StrictBlocksAndThreads) {
+ if (!LaunchArgs.Flags.StrictBlocksAndThreads) {
EffectiveNumThreads[0] =
getEffectiveNumThreads(GenericDevice, EffectiveNumThreads[0]);
EffectiveNumBlocks[0] = getEffectiveNumBlocks(
- GenericDevice, EffectiveNumBlocks[0], KernelArgs.Tripcount,
- EffectiveNumThreads[0], KernelArgs.UserThreadLimit[0] > 0);
+ GenericDevice, EffectiveNumBlocks[0], LaunchArgs.Tripcount,
+ EffectiveNumThreads[0], LaunchArgs.UserThreadLimit[0] > 0);
}
auto DynBlockMemConfOrErr = prepareBlockMemory(
- GenericDevice, KernelArgs,
+ GenericDevice, LaunchArgs,
EffectiveNumBlocks[0] * EffectiveNumBlocks[1] * EffectiveNumBlocks[2]);
if (!DynBlockMemConfOrErr)
return DynBlockMemConfOrErr.takeError();
@@ -292,26 +288,18 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
DynBlockMemConf.FallbackPtr);
auto KernelLaunchEnvOrErr =
- getKernelLaunchEnvironment(GenericDevice, KernelArgs, DynBlockMemConf,
+ getKernelLaunchEnvironment(GenericDevice, LaunchArgs, DynBlockMemConf,
AsyncInfoWrapper, EffectiveNumBlocks[0]);
if (!KernelLaunchEnvOrErr)
return KernelLaunchEnvOrErr.takeError();
- KernelLaunchParamsTy LaunchParams;
-
- // Kernel languages do not use the OpenMP indirection and argument parsing.
- if (KernelArgs.Flags.IsCUDA) {
- LaunchParams =
- *reinterpret_cast<KernelLaunchParamsTy *>(KernelArgs.ArgPtrs);
- } else if (KernelArgs.Flags.IsPtrArgs) {
- LaunchParams = KernelLaunchParamsTy{KernelArgs.NumArgs, KernelArgs.ArgPtrs};
- } else {
- LaunchParams =
- prepareArgs(GenericDevice, ArgPtrs, ArgOffsets, KernelArgs.NumArgs,
- Args, Ptrs, *KernelLaunchEnvOrErr, KernelArgs.Version);
- }
+ // Fill in the kernel launch environment (dyn_ptr) if this launch has a
+ // reserved slot for it. When replaying, getKernelLaunchEnvironment()
+ // returns null so the recorded value already in the slot is preserved.
+ if (LaunchArgs.DynPtrSlot && *KernelLaunchEnvOrErr)
+ *LaunchArgs.DynPtrSlot = *KernelLaunchEnvOrErr;
- if (auto Err = printLaunchInfo(GenericDevice, KernelArgs, EffectiveNumThreads,
+ if (auto Err = printLaunchInfo(GenericDevice, LaunchArgs, EffectiveNumThreads,
EffectiveNumBlocks))
return Err;
@@ -324,16 +312,16 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
// Record the kernel prologue data before kernel launch.
auto RRHandleOrErr = RecordReplay->recordPrologue(
- *this, KernelArgs, KernelExtraArgs, LaunchParams, EffectiveNumBlocks,
+ *this, LaunchArgs, KernelExtraArgs, EffectiveNumBlocks,
EffectiveNumThreads, DynBlockMemConf.NativeSize);
if (!RRHandleOrErr)
return RRHandleOrErr.takeError();
RRHandle = *RRHandleOrErr;
}
- if (auto Err = launchImpl(GenericDevice, EffectiveNumThreads,
- EffectiveNumBlocks, DynBlockMemConf.NativeSize,
- KernelArgs, LaunchParams, AsyncInfoWrapper))
+ if (auto Err =
+ launchImpl(GenericDevice, EffectiveNumThreads, EffectiveNumBlocks,
+ DynBlockMemConf.NativeSize, LaunchArgs, AsyncInfoWrapper))
return Err;
if (RecordReplay) {
@@ -347,41 +335,6 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
return Plugin::success();
}
-KernelLaunchParamsTy
-GenericKernelTy::prepareArgs(GenericDeviceTy &GenericDevice, void **ArgPtrs,
- ptrdiff_t *ArgOffsets, uint32_t &NumArgs,
- llvm::SmallVectorImpl<void *> &Args,
- llvm::SmallVectorImpl<void *> &Ptrs,
- KernelLaunchEnvironmentTy *KernelLaunchEnvironment,
- uint32_t Version) const {
- if (NumArgs == 0)
- return KernelLaunchParamsTy{};
-
- // The argument arrays already include the dyn_ptr slot at the end (appended
- // by the host for version >= 4, or by upgradeKernelArgs for version 3).
- Args.resize(NumArgs);
- Ptrs.resize(NumArgs);
-
- for (uint32_t I = 0; I < NumArgs; ++I)
- Args[I] = reinterpret_cast<void *>(reinterpret_cast<intptr_t>(ArgPtrs[I]) +
- ArgOffsets[I]);
-
- // Optionally assign the KernelLaunchEnvironment to the last slot (dyn_ptr).
- if (KernelLaunchEnvironment)
- Args[NumArgs - 1] = KernelLaunchEnvironment;
-
- // Version 3 device kernels have dyn_ptr baked in at position 0. Rotate the
- // last element to the front to match the device ABI.
- if (Version <= OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR &&
- KernelLaunchEnvironment)
- std::rotate(Args.begin(), Args.end() - 1, Args.end());
-
- for (uint32_t I = 0; I < NumArgs; ++I)
- Ptrs[I] = &Args[I];
-
- return KernelLaunchParamsTy{NumArgs, &Ptrs[0]};
-}
-
uint32_t
GenericKernelTy::getEffectiveNumThreads(GenericDeviceTy &GenericDevice,
uint32_t UserThreadLimit) const {
@@ -1176,9 +1129,8 @@ Error GenericDeviceTy::dataPrefetch(size_t Count, const void **Mems,
return Err;
}
-Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs,
- ptrdiff_t *ArgOffsets,
- KernelArgsTy &KernelArgs,
+Error GenericDeviceTy::launchKernel(void *EntryPtr,
+ KernelLaunchArgsTy &LaunchArgs,
KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfo) {
AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
@@ -1198,8 +1150,8 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs,
.emplace(&GenericKernel, std::move(StackTrace), AsyncInfo);
}
- auto Err = GenericKernel.launch(*this, ArgPtrs, ArgOffsets, KernelArgs,
- KernelExtraArgs, AsyncInfoWrapper);
+ auto Err = GenericKernel.launch(*this, LaunchArgs, KernelExtraArgs,
+ AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
@@ -1721,13 +1673,11 @@ int32_t GenericPluginTy::data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
}
int32_t GenericPluginTy::launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
- void **TgtArgs, ptrdiff_t *TgtOffsets,
- KernelArgsTy *KernelArgs,
+ KernelLaunchArgsTy &LaunchArgs,
KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfoPtr) {
- auto Err = getDevice(DeviceId).launchKernel(TgtEntryPtr, TgtArgs, TgtOffsets,
- *KernelArgs, KernelExtraArgs,
- AsyncInfoPtr);
+ auto Err = getDevice(DeviceId).launchKernel(TgtEntryPtr, LaunchArgs,
+ KernelExtraArgs, AsyncInfoPtr);
if (Err) {
REPORT() << "Failure to run target region " << TgtEntryPtr << " in device "
<< DeviceId << ": " << toString(std::move(Err));
diff --git a/offload/plugins-nextgen/common/src/RecordReplay.cpp b/offload/plugins-nextgen/common/src/RecordReplay.cpp
index dd03c83dca17e..9f170bfcc8f60 100644
--- a/offload/plugins-nextgen/common/src/RecordReplay.cpp
+++ b/offload/plugins-nextgen/common/src/RecordReplay.cpp
@@ -167,9 +167,8 @@ Expected<void *> RecordReplayTy::allocate(uint64_t Size) {
Error RecordReplayTy::deallocate(void *Ptr) { return Plugin::success(); }
Expected<RecordReplayTy::HandleTy> RecordReplayTy::recordPrologue(
- const GenericKernelTy &Kernel, const KernelArgsTy &KernelArgs,
- const KernelExtraArgsTy *KernelExtraArgs,
- const KernelLaunchParamsTy &LaunchParams, uint32_t NumTeams[3],
+ const GenericKernelTy &Kernel, const KernelLaunchArgsTy &LaunchArgs,
+ const KernelExtraArgsTy *KernelExtraArgs, uint32_t NumTeams[3],
uint32_t NumThreads[3], uint32_t SharedMemorySize) {
if (!isRecordingOrReplaying())
return HandleTy{nullptr, false};
@@ -184,11 +183,10 @@ Expected<RecordReplayTy::HandleTy> RecordReplayTy::recordPrologue(
return Handle;
if (isRecording()) {
- if (auto Err = recordDescImpl(Kernel, Instance, KernelArgs, LaunchParams))
+ if (auto Err = recordDescImpl(Kernel, Instance, LaunchArgs))
return Err;
- if (auto Err =
- recordPrologueImpl(Kernel, Instance, KernelArgs, LaunchParams))
+ if (auto Err = recordPrologueImpl(Kernel, Instance, LaunchArgs))
return Err;
}
@@ -235,7 +233,7 @@ void RecordReplayTy::populateReplayOutcome(const InstanceTy &Instance,
Error NativeRecordReplayTy::recordPrologueImpl(
const GenericKernelTy &Kernel, const InstanceTy &Instance,
- const KernelArgsTy &KernelArgs, const KernelLaunchParamsTy &LaunchParams) {
+ const KernelLaunchArgsTy &LaunchArgs) {
SmallString<128> SnapshotFilename =
getFilename(Instance, FileTy::PrologueSnapshot);
if (auto Err = recordSnapshot(SnapshotFilename.c_str()))
@@ -260,21 +258,21 @@ Error NativeRecordReplayTy::recordEpilogueImpl(const GenericKernelTy &Kernel,
Error NativeRecordReplayTy::recordDescImpl(
const GenericKernelTy &Kernel, const InstanceTy &Instance,
- const KernelArgsTy &KernelArgs, const KernelLaunchParamsTy &LaunchParams) {
+ const KernelLaunchArgsTy &LaunchArgs) {
json::Object JsonKernelInfo;
JsonKernelInfo["Name"] = Kernel.getName();
- JsonKernelInfo["NumArgs"] = KernelArgs.NumArgs;
+ JsonKernelInfo["NumArgs"] = LaunchArgs.NumArgs;
JsonKernelInfo["NumTeams"] = Instance.NumTeams;
JsonKernelInfo["NumThreads"] = Instance.NumThreads;
JsonKernelInfo["SharedMemorySize"] = Instance.SharedMemorySize;
- JsonKernelInfo["LoopTripCount"] = KernelArgs.Tripcount;
+ JsonKernelInfo["LoopTripCount"] = LaunchArgs.Tripcount;
JsonKernelInfo["DeviceId"] = Device.getDeviceId();
JsonKernelInfo["VAllocAddr"] = (intptr_t)StartAddr;
JsonKernelInfo["VAllocSize"] = TotalSize;
// Export minimum and maximum for allowed number of teams. If zero, it means
// there was no restriction provided by the program.
- uint32_t MinMaxBlocks = std::max(KernelArgs.UserNumBlocks[0], uint32_t(0));
+ uint32_t MinMaxBlocks = std::max(LaunchArgs.UserNumBlocks[0], uint32_t(0));
json::Array JsonTeamsLimits;
JsonTeamsLimits.push_back(MinMaxBlocks);
JsonTeamsLimits.push_back(MinMaxBlocks);
@@ -282,7 +280,7 @@ Error NativeRecordReplayTy::recordDescImpl(
// Export minimum and maximum for allowed number of threads. If zero, it means
// there was no restriction provided by the program.
- uint32_t UserThreads = std::max(KernelArgs.UserThreadLimit[0], uint32_t(0));
+ uint32_t UserThreads = std::max(LaunchArgs.UserThreadLimit[0], uint32_t(0));
uint32_t MaxThreads = UserThreads
? std::min(UserThreads, Kernel.getMaxThreads())
: Kernel.getMaxThreads();
@@ -292,12 +290,12 @@ Error NativeRecordReplayTy::recordDescImpl(
JsonKernelInfo["ThreadsLimits"] = json::Value(std::move(JsonThreadsLimits));
json::Array JsonArgPtrs;
- for (uint32_t I = 0; I < KernelArgs.NumArgs; ++I)
- JsonArgPtrs.push_back((intptr_t)(*(void **)LaunchParams.Args[I]));
+ for (uint32_t I = 0; I < LaunchArgs.NumArgs; ++I)
+ JsonArgPtrs.push_back((intptr_t)(*(void **)LaunchArgs.Args[I]));
JsonKernelInfo["ArgPtrs"] = json::Value(std::move(JsonArgPtrs));
json::Array JsonArgOffsets;
- for (uint32_t I = 0; I < KernelArgs.NumArgs; ++I)
+ for (uint32_t I = 0; I < LaunchArgs.NumArgs; ++I)
JsonArgOffsets.push_back(0);
JsonKernelInfo["ArgOffsets"] = json::Value(std::move(JsonArgOffsets));
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 9bf835b63813c..9fc4a8b0f208f 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -139,7 +139,7 @@ struct CUDAKernelTy : public GenericKernelTy {
/// Launch the CUDA kernel function.
Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
uint32_t NumBlocks[3], uint32_t DynBlockMemSize,
- KernelArgsTy &KernelArgs, KernelLaunchParamsTy LaunchParams,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
/// Return maximum block size for maximum occupancy
@@ -1481,11 +1481,11 @@ struct CUDADeviceTy : public GenericDeviceTy {
AsyncInfoWrapperTy AsyncInfoWrapper(*this, nullptr);
- KernelArgsTy KernelArgs = {};
+ KernelLaunchArgsTy LaunchArgs = {};
uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
- auto Err = CUDAKernel.launchImpl(*this, NumBlocksAndThreads,
- NumBlocksAndThreads, 0, KernelArgs,
- KernelLaunchParamsTy{}, AsyncInfoWrapper);
+ auto Err =
+ CUDAKernel.launchImpl(*this, NumBlocksAndThreads, NumBlocksAndThreads,
+ 0, LaunchArgs, AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
if (Err)
@@ -1530,8 +1530,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
uint32_t DynBlockMemSize,
- KernelArgsTy &KernelArgs,
- KernelLaunchParamsTy LaunchParams,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const {
CUDADeviceTy &CUDADevice = static_cast<CUDADeviceTy &>(GenericDevice);
@@ -1557,7 +1556,7 @@ Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
CUlaunchAttribute CoopAttr;
CoopAttr.id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE;
- CoopAttr.value.cooperative = KernelArgs.Flags.Cooperative;
+ CoopAttr.value.cooperative = LaunchArgs.Flags.Cooperative;
CUlaunchConfig LaunchConfig = {NumBlocks[0], NumBlocks[1],
NumBlocks[2], NumThreads[0],
@@ -1565,7 +1564,7 @@ Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
DynBlockMemSize, Stream,
&CoopAttr, 1};
- CUresult Res = cuLaunchKernelEx(&LaunchConfig, Func, LaunchParams.Args,
+ CUresult Res = cuLaunchKernelEx(&LaunchConfig, Func, LaunchArgs.Args,
/*extra=*/nullptr);
// Register a callback to indicate when the kernel is complete.
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 95ec3820f2657..e75bd2cba9e85 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -88,18 +88,18 @@ struct GenELF64KernelTy : public GenericKernelTy {
/// Launch the kernel using the arguments.
Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
uint32_t NumBlocks[3], uint32_t DynBlockMemSize,
- KernelArgsTy &KernelArgs, KernelLaunchParamsTy LaunchParams,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const override {
- if (KernelArgs.Version < OMP_KERNEL_ARG_VERSION)
+ if (LaunchArgs.OmpABIVersion < OMP_KERNEL_ARG_VERSION)
return Plugin::error(ErrorCode::UNSUPPORTED,
"Incompatible kernel argument version for plugin");
// Cooperative kernel launch is not supported for host
- if (KernelArgs.Flags.Cooperative)
+ if (LaunchArgs.Flags.Cooperative)
return Plugin::error(ErrorCode::UNSUPPORTED,
"cooperative kernel launch not supported for host");
// TODO: The data will need to be copied locally if we ever support
// asynchronous kernel launches in the host interface.
- Func(LaunchParams.Args);
+ Func(LaunchArgs.Args);
return Plugin::success();
}
diff --git a/offload/plugins-nextgen/level_zero/include/L0Kernel.h b/offload/plugins-nextgen/level_zero/include/L0Kernel.h
index 5630dfe4ba585..4ab376ed15e63 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Kernel.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Kernel.h
@@ -45,10 +45,10 @@ struct L0LaunchEnvTy {
void **ArgPtrs = nullptr;
std::unique_lock<std::mutex> Lock;
- L0LaunchEnvTy(KernelPropertiesTy &KernelPR, KernelArgsTy &KernelArgs,
- KernelLaunchParamsTy LaunchParams)
- : KernelPR(KernelPR), IsCooperative(KernelArgs.Flags.Cooperative),
- ArgPtrs(LaunchParams.Args), Lock(KernelPR.Mtx, std::defer_lock) {}
+ L0LaunchEnvTy(KernelPropertiesTy &KernelPR,
+ const KernelLaunchArgsTy &LaunchArgs)
+ : KernelPR(KernelPR), IsCooperative(LaunchArgs.Flags.Cooperative),
+ ArgPtrs(LaunchArgs.Args), Lock(KernelPR.Mtx, std::defer_lock) {}
};
class L0KernelTy : public GenericKernelTy {
@@ -81,7 +81,7 @@ class L0KernelTy : public GenericKernelTy {
/// Launch the L0 kernel function.
Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
uint32_t NumBlocks[3], uint32_t DynBlockMemSize,
- KernelArgsTy &KernelArgs, KernelLaunchParamsTy LaunchParams,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
Error deinit() {
CALL_ZE_RET_ERROR(zeKernelDestroy, zeKernel);
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index 076dfa080f86e..42559423c60ab 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -957,11 +957,11 @@ Error L0DeviceTy::callGlobalCtorDtorCommon(GenericPluginTy &Plugin,
AsyncInfoWrapperTy AsyncInfoWrapper(*this, /*AsyncInfoPtr=*/nullptr);
- KernelArgsTy KernelArgs{};
+ KernelLaunchArgsTy LaunchArgs{};
uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
auto Err =
L0Kernel.launchImpl(*this, NumBlocksAndThreads, NumBlocksAndThreads, 0,
- KernelArgs, KernelLaunchParamsTy{}, AsyncInfoWrapper);
+ LaunchArgs, AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
return CleanupBufferAndErr(std::move(Err));
diff --git a/offload/plugins-nextgen/level_zero/src/L0Kernel.cpp b/offload/plugins-nextgen/level_zero/src/L0Kernel.cpp
index 2b5d63cc73fb3..bd0e21d2301e9 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Kernel.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Kernel.cpp
@@ -132,8 +132,8 @@ Error L0KernelTy::setIndirectFlags(L0DeviceTy &L0Device,
Error L0KernelTy::launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
- uint32_t DynBlockMemSize, KernelArgsTy &KernelArgs,
- KernelLaunchParamsTy LaunchParams,
+ uint32_t DynBlockMemSize,
+ KernelLaunchArgsTy &LaunchArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const {
if (DynBlockMemSize > 0)
return Plugin::error(ErrorCode::UNSUPPORTED,
@@ -149,7 +149,7 @@ Error L0KernelTy::launchImpl(GenericDeviceTy &GenericDevice,
DPxPTR(zeKernel));
auto *IdStr = L0Device.getZeIdCStr();
- bool IsCooperative = KernelArgs.Flags.Cooperative;
+ bool IsCooperative = LaunchArgs.Flags.Cooperative;
if (IsCooperative && !L0Device.supportsCooperativeKernels()) {
return Plugin::error(
@@ -162,7 +162,7 @@ Error L0KernelTy::launchImpl(GenericDeviceTy &GenericDevice,
auto *Queue = *QueueOrErr;
auto &KernelPR = getProperties();
- L0LaunchEnvTy KEnv(KernelPR, KernelArgs, LaunchParams);
+ L0LaunchEnvTy KEnv(KernelPR, LaunchArgs);
// Protect from kernel preparation to submission as kernels are shared.
KEnv.Lock.lock();
@@ -195,12 +195,12 @@ Error L0KernelTy::launchImpl(GenericDeviceTy &GenericDevice,
// With pointer-array arguments, zeCommandListAppendLaunchKernelWithArguments
// folds group-size, per-argument set, and launch into a single call.
- if (LaunchParams.NumArgs != KernelPR.NumKernelArgs)
+ if (LaunchArgs.NumArgs != KernelPR.NumKernelArgs)
return Plugin::error(
ErrorCode::INVALID_ARGUMENT,
"Number of arguments (%u) does not match the number of arguments "
"expected by the kernel (%u)",
- LaunchParams.NumArgs, KernelPR.NumKernelArgs);
+ LaunchArgs.NumArgs, KernelPR.NumKernelArgs);
if (auto Err = setIndirectFlags(L0Device, KEnv))
return Err;
>From a136d3070dd1300b5b5db4d5bc0345742857e151 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Tue, 4 Aug 2026 02:49:55 -0700
Subject: [PATCH 2/6] Rotate ArgSizes too
---
offload/libomptarget/device.cpp | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 8bdde45dfc494..785d67e651259 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -358,7 +358,7 @@ int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
/// Resolve \p NumArgs (base pointer, offset) pairs into a flattened array of
/// argument-value pointers suitable for a kernel launch, writing the result
-/// into \p LaunchArgs.NumArgs/Args.
+/// into \p LaunchArgs.NumArgs/Args.
static void resolveKernelLaunchParams(void **TgtArgs, ptrdiff_t *TgtOffsets,
uint32_t NumArgs,
llvm::SmallVector<void *> &Args,
@@ -386,6 +386,7 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
KernelExtraArgsTy *KernelExtraArgs,
AsyncInfoTy &AsyncInfo) {
llvm::SmallVector<void *> Args, Ptrs;
+ llvm::SmallVector<int64_t> ArgSizes;
KernelLaunchArgsTy LaunchArgs;
LaunchArgs.OmpABIVersion = KernelArgs.Version;
@@ -421,6 +422,14 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
if (KernelArgs.Version == OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) {
std::rotate(Args.begin(), Args.end() - 1, Args.end());
LaunchArgs.DynPtrSlot = &Args[0];
+
+ // Keep ArgSizes in sync with the rotated Args, if present.
+ if (LaunchArgs.ArgSizes) {
+ ArgSizes.assign(LaunchArgs.ArgSizes,
+ LaunchArgs.ArgSizes + KernelArgs.NumArgs);
+ std::rotate(ArgSizes.begin(), ArgSizes.end() - 1, ArgSizes.end());
+ LaunchArgs.ArgSizes = ArgSizes.data();
+ }
} else {
LaunchArgs.DynPtrSlot = &Args[KernelArgs.NumArgs - 1];
}
>From 8825c1514bac24cb8f8f55407475daf5e5c7d4f6 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Wed, 5 Aug 2026 08:52:24 -0700
Subject: [PATCH 3/6] use llvm::copy
---
offload/libomptarget/device.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 785d67e651259..2ab95167b983a 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -393,10 +393,8 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
LaunchArgs.ArgSizes = KernelArgs.ArgSizes;
LaunchArgs.Tripcount = KernelArgs.Tripcount;
LaunchArgs.DynCGroupMem = KernelArgs.DynCGroupMem;
- std::copy(std::begin(KernelArgs.UserNumBlocks),
- std::end(KernelArgs.UserNumBlocks), LaunchArgs.UserNumBlocks);
- std::copy(std::begin(KernelArgs.UserThreadLimit),
- std::end(KernelArgs.UserThreadLimit), LaunchArgs.UserThreadLimit);
+ llvm::copy(KernelArgs.UserNumBlocks, LaunchArgs.UserNumBlocks);
+ llvm::copy(KernelArgs.UserThreadLimit, LaunchArgs.UserThreadLimit);
LaunchArgs.Flags.Cooperative = KernelArgs.Flags.Cooperative;
LaunchArgs.Flags.StrictBlocksAndThreads =
KernelArgs.Flags.StrictBlocksAndThreads;
>From 363756a4955fb63f32c4a46bea6709bde95afb5d Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Thu, 6 Aug 2026 00:55:59 -0700
Subject: [PATCH 4/6] Merge KernelExtraArgsTy into KernelLaunchArgsTy
---
offload/include/Shared/APITypes.h | 7 -------
offload/include/device.h | 2 +-
offload/liboffload/src/OffloadImpl.cpp | 3 +--
offload/libomptarget/device.cpp | 10 +++++-----
offload/libomptarget/omptarget.cpp | 5 +----
.../common/include/PluginInterface.h | 6 +++---
.../plugins-nextgen/common/include/RecordReplay.h | 1 -
.../plugins-nextgen/common/src/PluginInterface.cpp | 14 +++++---------
.../plugins-nextgen/common/src/RecordReplay.cpp | 9 ++++-----
9 files changed, 20 insertions(+), 37 deletions(-)
diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 47d8c49bf7ef2..71cf6773437d1 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -143,13 +143,6 @@ struct KernelReplayOutcomeTy {
/// reused for future replays of the same kernel.
void *ReplayDeviceAlloc = nullptr;
};
-
-/// Extra kernel arguments managed by the runtime components. Notice these
-/// arguments are additional to the ones in KernelArgsTy, which are usually
-/// generated by the compiler.
-struct KernelExtraArgsTy {
- KernelReplayOutcomeTy *ReplayOutcome = nullptr;
-};
}
#endif // OMPTARGET_SHARED_API_TYPES_H
diff --git a/offload/include/device.h b/offload/include/device.h
index af103c316c3cf..266a2a675df0c 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -117,7 +117,7 @@ struct DeviceTy {
// Launch the kernel identified by \p TgtEntryPtr with the given arguments.
int32_t launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs,
- KernelExtraArgsTy *KernelExtraArgs,
+ KernelReplayOutcomeTy *ReplayOutcome,
AsyncInfoTy &AsyncInfo);
/// Synchronize device/queue/event based on \p AsyncInfo and return
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 277df26fbce8a..6cf1579607fad 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -1270,8 +1270,7 @@ Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,
AsyncInfoWrapperTy AsyncInfoWrapper(*DeviceImpl, QueueImpl);
auto *KernelImpl = std::get<GenericKernelTy *>(Kernel->PluginImpl);
- auto Err =
- KernelImpl->launch(*DeviceImpl, LaunchArgs, nullptr, AsyncInfoWrapper);
+ auto Err = KernelImpl->launch(*DeviceImpl, LaunchArgs, AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
if (Err)
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 2ab95167b983a..9c4575c939b02 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -371,11 +371,11 @@ static void resolveKernelLaunchParams(void **TgtArgs, ptrdiff_t *TgtOffsets,
Args.resize(NumArgs);
Ptrs.resize(NumArgs);
- for (uint32_t I = 0; I < NumArgs; ++I)
+ for (uint32_t I = 0; I < NumArgs; ++I) {
Args[I] = reinterpret_cast<void *>(reinterpret_cast<intptr_t>(TgtArgs[I]) +
TgtOffsets[I]);
- for (uint32_t I = 0; I < NumArgs; ++I)
Ptrs[I] = &Args[I];
+ }
LaunchArgs.Args = &Ptrs[0];
}
@@ -383,13 +383,14 @@ static void resolveKernelLaunchParams(void **TgtArgs, ptrdiff_t *TgtOffsets,
// Run region on device
int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs,
- KernelExtraArgsTy *KernelExtraArgs,
+ KernelReplayOutcomeTy *ReplayOutcome,
AsyncInfoTy &AsyncInfo) {
llvm::SmallVector<void *> Args, Ptrs;
llvm::SmallVector<int64_t> ArgSizes;
KernelLaunchArgsTy LaunchArgs;
LaunchArgs.OmpABIVersion = KernelArgs.Version;
+ LaunchArgs.ReplayOutcome = ReplayOutcome;
LaunchArgs.ArgSizes = KernelArgs.ArgSizes;
LaunchArgs.Tripcount = KernelArgs.Tripcount;
LaunchArgs.DynCGroupMem = KernelArgs.DynCGroupMem;
@@ -434,8 +435,7 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
}
}
- return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, LaunchArgs,
- KernelExtraArgs, AsyncInfo);
+ return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, LaunchArgs, AsyncInfo);
}
// Run region on device
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 27ee173be06f9..615097748c3c9 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -2490,11 +2490,8 @@ int target_replay(ident_t *Loc, DeviceTy &Device, void *HostPtr,
KernelArgs.DynCGroupMem = SharedMemorySize;
KernelArgs.Flags.StrictBlocksAndThreads = true;
- KernelExtraArgsTy KernelExtraArgs{};
- KernelExtraArgs.ReplayOutcome = ReplayOutcome;
-
int Ret = Device.launchKernel(Symbols[0].DevPtr, TgtArgs, TgtOffsets,
- KernelArgs, &KernelExtraArgs, AsyncInfo);
+ KernelArgs, ReplayOutcome, AsyncInfo);
if (Ret != OFFLOAD_SUCCESS) {
REPORT() << "Failed to launch kernel replay.";
return OFFLOAD_FAIL;
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index a669cddd0b76e..389e8d3e3c804 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -461,6 +461,9 @@ struct KernelLaunchArgsTy {
uint64_t DynCGroupMemFallback : 2; // The fallback for dynamic cgroup mem.
uint64_t Unused : 60;
} Flags = {0, 0, 0, 0};
+ /// Set by the caller when replaying a previously recorded kernel launch, so
+ /// the plugin can report the outcome back; null for a normal launch.
+ KernelReplayOutcomeTy *ReplayOutcome = nullptr;
};
/// Class implementing common functionalities of offload kernels. Each plugin
@@ -485,7 +488,6 @@ struct GenericKernelTy {
/// of it reserved for the kernel launch environment (dyn_ptr); the caller
/// owns the storage it points into.
Error launch(GenericDeviceTy &GenericDevice, KernelLaunchArgsTy &LaunchArgs,
- KernelExtraArgsTy *KernelExtraArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const;
virtual Error launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
@@ -1134,7 +1136,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
/// Run the kernel associated with \p EntryPtr
Error launchKernel(void *EntryPtr, KernelLaunchArgsTy &LaunchArgs,
- KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfo);
/// Initialize a __tgt_async_info structure.
@@ -1749,7 +1750,6 @@ struct GenericPluginTy {
/// Begin executing a kernel on the given device.
int32_t launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
KernelLaunchArgsTy &LaunchArgs,
- KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfoPtr);
/// Synchronize an asyncrhonous queue with the plugin runtime.
diff --git a/offload/plugins-nextgen/common/include/RecordReplay.h b/offload/plugins-nextgen/common/include/RecordReplay.h
index 5402bb64d8b24..8a06dcb200e84 100644
--- a/offload/plugins-nextgen/common/include/RecordReplay.h
+++ b/offload/plugins-nextgen/common/include/RecordReplay.h
@@ -207,7 +207,6 @@ struct RecordReplayTy {
/// instance is registered.
Expected<HandleTy> recordPrologue(const GenericKernelTy &Kernel,
const KernelLaunchArgsTy &LaunchArgs,
- const KernelExtraArgsTy *KernelExtraArgs,
uint32_t NumTeams[3],
uint32_t NumThreads[3],
uint32_t SharedMemorySize);
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index e283e6c97ec72..c91258c1fbec8 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -244,7 +244,6 @@ GenericKernelTy::prepareBlockMemory(GenericDeviceTy &GenericDevice,
Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice,
KernelLaunchArgsTy &LaunchArgs,
- KernelExtraArgsTy *KernelExtraArgs,
AsyncInfoWrapperTy &AsyncInfoWrapper) const {
uint32_t EffectiveNumThreads[3] = {LaunchArgs.UserThreadLimit[0],
LaunchArgs.UserThreadLimit[1],
@@ -312,8 +311,8 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice,
// Record the kernel prologue data before kernel launch.
auto RRHandleOrErr = RecordReplay->recordPrologue(
- *this, LaunchArgs, KernelExtraArgs, EffectiveNumBlocks,
- EffectiveNumThreads, DynBlockMemConf.NativeSize);
+ *this, LaunchArgs, EffectiveNumBlocks, EffectiveNumThreads,
+ DynBlockMemConf.NativeSize);
if (!RRHandleOrErr)
return RRHandleOrErr.takeError();
RRHandle = *RRHandleOrErr;
@@ -1131,7 +1130,6 @@ Error GenericDeviceTy::dataPrefetch(size_t Count, const void **Mems,
Error GenericDeviceTy::launchKernel(void *EntryPtr,
KernelLaunchArgsTy &LaunchArgs,
- KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfo) {
AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
@@ -1150,8 +1148,7 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr,
.emplace(&GenericKernel, std::move(StackTrace), AsyncInfo);
}
- auto Err = GenericKernel.launch(*this, LaunchArgs, KernelExtraArgs,
- AsyncInfoWrapper);
+ auto Err = GenericKernel.launch(*this, LaunchArgs, AsyncInfoWrapper);
AsyncInfoWrapper.finalize(Err);
@@ -1674,10 +1671,9 @@ int32_t GenericPluginTy::data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
int32_t GenericPluginTy::launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
KernelLaunchArgsTy &LaunchArgs,
- KernelExtraArgsTy *KernelExtraArgs,
__tgt_async_info *AsyncInfoPtr) {
- auto Err = getDevice(DeviceId).launchKernel(TgtEntryPtr, LaunchArgs,
- KernelExtraArgs, AsyncInfoPtr);
+ auto Err =
+ getDevice(DeviceId).launchKernel(TgtEntryPtr, LaunchArgs, AsyncInfoPtr);
if (Err) {
REPORT() << "Failure to run target region " << TgtEntryPtr << " in device "
<< DeviceId << ": " << toString(std::move(Err));
diff --git a/offload/plugins-nextgen/common/src/RecordReplay.cpp b/offload/plugins-nextgen/common/src/RecordReplay.cpp
index 9f170bfcc8f60..c36102804e3b8 100644
--- a/offload/plugins-nextgen/common/src/RecordReplay.cpp
+++ b/offload/plugins-nextgen/common/src/RecordReplay.cpp
@@ -168,15 +168,14 @@ Error RecordReplayTy::deallocate(void *Ptr) { return Plugin::success(); }
Expected<RecordReplayTy::HandleTy> RecordReplayTy::recordPrologue(
const GenericKernelTy &Kernel, const KernelLaunchArgsTy &LaunchArgs,
- const KernelExtraArgsTy *KernelExtraArgs, uint32_t NumTeams[3],
- uint32_t NumThreads[3], uint32_t SharedMemorySize) {
+ uint32_t NumTeams[3], uint32_t NumThreads[3], uint32_t SharedMemorySize) {
if (!isRecordingOrReplaying())
return HandleTy{nullptr, false};
// Register the instance and avoid recording if it is inactive or replaying.
- auto [Instance, First] = registerInstance(
- Kernel, NumTeams[0], NumThreads[0], SharedMemorySize,
- (KernelExtraArgs) ? KernelExtraArgs->ReplayOutcome : nullptr);
+ auto [Instance, First] =
+ registerInstance(Kernel, NumTeams[0], NumThreads[0], SharedMemorySize,
+ LaunchArgs.ReplayOutcome);
HandleTy Handle{&Instance, First};
if (!First)
>From 5d915ab315f208e77245974b974aa81ee01846b7 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Thu, 6 Aug 2026 03:04:56 -0700
Subject: [PATCH 5/6] adress feedback
---
offload/libomptarget/device.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 9c4575c939b02..3631928da94fb 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -359,18 +359,19 @@ int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
/// Resolve \p NumArgs (base pointer, offset) pairs into a flattened array of
/// argument-value pointers suitable for a kernel launch, writing the result
/// into \p LaunchArgs.NumArgs/Args.
-static void resolveKernelLaunchParams(void **TgtArgs, ptrdiff_t *TgtOffsets,
+static void resolveKernelLaunchParams(void ** const TgtArgs,
+ ptrdiff_t * const TgtOffsets,
uint32_t NumArgs,
llvm::SmallVector<void *> &Args,
llvm::SmallVector<void *> &Ptrs,
KernelLaunchArgsTy &LaunchArgs) {
LaunchArgs.NumArgs = NumArgs;
- if (NumArgs == 0)
- return;
-
Args.resize(NumArgs);
Ptrs.resize(NumArgs);
+ if (NumArgs == 0)
+ return;
+
for (uint32_t I = 0; I < NumArgs; ++I) {
Args[I] = reinterpret_cast<void *>(reinterpret_cast<intptr_t>(TgtArgs[I]) +
TgtOffsets[I]);
>From 5da1282771daa7603c822a09bf1113c6ed8f2d64 Mon Sep 17 00:00:00 2001
From: "Duran, Alex" <alejandro.duran at intel.com>
Date: Thu, 6 Aug 2026 03:42:36 -0700
Subject: [PATCH 6/6] format
---
offload/libomptarget/device.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 3631928da94fb..29bb1f4a828c8 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -359,8 +359,8 @@ int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
/// Resolve \p NumArgs (base pointer, offset) pairs into a flattened array of
/// argument-value pointers suitable for a kernel launch, writing the result
/// into \p LaunchArgs.NumArgs/Args.
-static void resolveKernelLaunchParams(void ** const TgtArgs,
- ptrdiff_t * const TgtOffsets,
+static void resolveKernelLaunchParams(void **const TgtArgs,
+ ptrdiff_t *const TgtOffsets,
uint32_t NumArgs,
llvm::SmallVector<void *> &Args,
llvm::SmallVector<void *> &Ptrs,
More information about the llvm-commits
mailing list