[llvm] [Offload] Add GenericProfilerTy abstraction and APITypes extensions (PR #225639)
Jan Patrick Lehr via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 03:21:24 PDT 2026
https://github.com/jplehr updated https://github.com/llvm/llvm-project/pull/225639
>From 4efb650f5ab9a2b9206b4b9369804637e124be5d Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:27:58 -0500
Subject: [PATCH] [Offload] Add GenericProfilerTy abstraction and APITypes
extensions
Introduce GenericProfilerTy alongside the existing OMPT callback dispatch.
The weak profiler factory returns a no-op implementation, so the new hooks
are silent while the established callback path continues to handle OMPT
device events.
Co-Authored-By: Dhruva Chakrabarti <dhruva.chakrabarti at amd.com>
Co-Authored-By: Michael Halkenhauser <michaelgerald.halkenhauser at amd.com>
Assisted-by: Claude Code
---
offload/include/Shared/APITypes.h | 5 +
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 15 +-
offload/plugins-nextgen/common/CMakeLists.txt | 1 +
.../common/include/GenericProfiler.h | 197 ++++++++++++++++++
.../common/include/PluginInterface.h | 62 ++++--
.../common/src/GenericProfiler.cpp | 36 ++++
.../common/src/PluginInterface.cpp | 119 +++++++----
offload/plugins-nextgen/cuda/src/rtl.cpp | 3 +-
offload/plugins-nextgen/host/src/rtl.cpp | 13 +-
.../level_zero/include/L0Device.h | 3 +-
.../level_zero/src/L0Device.cpp | 3 +-
11 files changed, 385 insertions(+), 72 deletions(-)
create mode 100644 offload/plugins-nextgen/common/include/GenericProfiler.h
create mode 100644 offload/plugins-nextgen/common/src/GenericProfiler.cpp
diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 7a151ab4ee6121..7ded9742942399 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -88,6 +88,11 @@ struct __tgt_async_info {
/// ensure it is a valid location while the transfer to the device is
/// happening.
KernelLaunchEnvironmentTy KernelLaunchEnvironment;
+
+ /// Opaque handle for profiler-specific data (e.g., OMPT trace record info).
+ /// Owned by the profiler; the runtime threads this pointer through the plugin
+ /// layer to associate async operations with trace records.
+ void *ProfilerData = nullptr;
};
/// This struct contains all of the arguments to a target kernel region launch.
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 92afb883840ef6..82baa7153a247a 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2282,7 +2282,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
~AMDGPUDeviceTy() {}
/// Initialize the device, its resources and get its properties.
- Error initImpl(GenericPluginTy &Plugin) override {
+ Error initImpl(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) override {
// First setup all the memory pools.
if (auto Err = initMemoryPools())
return Err;
@@ -3989,9 +3990,9 @@ struct AMDGPUPluginContextTy final : public PluginContextTy {
Expected<void *> allocate(GenericDeviceTy &Device, int64_t Size,
void *HostPtr, TargetAllocTy Kind,
- size_t Alignment) override;
+ size_t Alignment, GenericProfilerTy *ProfilerPtr) override;
Error deallocate(GenericDeviceTy &Device, void *Ptr,
- TargetAllocTy Kind) override;
+ TargetAllocTy Kind, GenericProfilerTy *ProfilerPtr) override;
Expected<PluginAllocInfoTy> getAllocInfo(const void *Ptr) override;
private:
@@ -4316,9 +4317,9 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
Expected<void *> AMDGPUPluginContextTy::allocate(GenericDeviceTy &Device,
int64_t Size, void *HostPtr,
TargetAllocTy Kind,
- size_t Alignment) {
+ size_t Alignment, GenericProfilerTy *ProfilerPtr) {
auto PtrOrErr =
- PluginContextTy::allocate(Device, Size, HostPtr, Kind, Alignment);
+ PluginContextTy::allocate(Device, Size, HostPtr, Kind, Alignment, ProfilerPtr);
if (!PtrOrErr || !*PtrOrErr)
return PtrOrErr;
std::lock_guard<std::mutex> Lock(AllocationsMutex);
@@ -4327,7 +4328,7 @@ Expected<void *> AMDGPUPluginContextTy::allocate(GenericDeviceTy &Device,
}
Error AMDGPUPluginContextTy::deallocate(GenericDeviceTy &Device, void *Ptr,
- TargetAllocTy Kind) {
+ TargetAllocTy Kind, GenericProfilerTy *ProfilerPtr) {
// Erase before base deallocate: once Ptr returns to the MM freelist a
// concurrent alloc could reuse it and re-populate Allocations. On failure
// Ptr is in an undetermined state (maybe freed, maybe not) so we don't
@@ -4336,7 +4337,7 @@ Error AMDGPUPluginContextTy::deallocate(GenericDeviceTy &Device, void *Ptr,
std::lock_guard<std::mutex> Lock(AllocationsMutex);
Allocations.erase(Ptr);
}
- return PluginContextTy::deallocate(Device, Ptr, Kind);
+ return PluginContextTy::deallocate(Device, Ptr, Kind, ProfilerPtr);
}
Expected<PluginAllocInfoTy>
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 6ad20796c3ca45..19ce102309a724 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,6 +11,7 @@ add_public_tablegen_target(PluginErrcodes)
# don't want to export `PluginInterface` while `add_llvm_library` requires that.
add_library(PluginCommon OBJECT
src/PluginInterface.cpp
+ src/GenericProfiler.cpp
src/GlobalHandler.cpp
src/JIT.cpp
src/RecordReplay.cpp
diff --git a/offload/plugins-nextgen/common/include/GenericProfiler.h b/offload/plugins-nextgen/common/include/GenericProfiler.h
new file mode 100644
index 00000000000000..57bb0847f32d91
--- /dev/null
+++ b/offload/plugins-nextgen/common/include/GenericProfiler.h
@@ -0,0 +1,197 @@
+//===- GenericProfiler.h - GenericProfiler interface for use in Plugins ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The GenericProfiler interface allows to implement profiler logic for various
+// backends, such as OMPT or other tracing mechanisms.
+// This enables the plugins to be agnostic of the actual high-level language
+// that is implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+
+#include "Shared/APITypes.h"
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+
+class OmptTracingBufferMgr;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+template <typename FunT, typename... ArgsT, size_t... IdxSequence>
+void callViaIndexSeq(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+ uint64_t EndNanos, std::tuple<ArgsT...> Args,
+ std::index_sequence<IdxSequence...>) {
+ F(P, StartNanos, EndNanos, std::get<IdxSequence>(Args)...);
+}
+
+template <typename FunT, typename... ArgsT>
+void callViaUnpack(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+ uint64_t EndNanos, std::tuple<ArgsT...> Tup) {
+ callViaIndexSeq(F, P, StartNanos, EndNanos, Tup,
+ std::index_sequence_for<ArgsT...>{});
+}
+
+/// Abstraction layer to implement different profiler backends.
+///
+/// The plugins call into the GenericProfilerTy to handle the specific events
+/// with whatever specific backend was instantiated. For now, the supported
+/// backends are limited to an OMPT implementation.
+class GenericProfilerTy {
+public:
+ GenericProfilerTy() = default;
+ virtual ~GenericProfilerTy() = default;
+
+ /// Obtain a pointer to profiler-specific data, if any.
+ virtual void *getProfilerSpecificData() { return nullptr; }
+
+ /// Obtain the trace record manager owned by this profiler, if any. Profilers
+ /// that produce trace records own the manager so that the records handed out
+ /// cannot outlive the buffers they point into.
+ virtual OmptTracingBufferMgr *getTraceRecordManager() { return nullptr; }
+
+ virtual bool isProfilingEnabled() { return false; }
+
+ /// Set the factors which are used to interpolate the device clock compared to
+ /// the host clock. This follows a simple linear interpolation: Slope * <time>
+ /// + Offset.
+ void setTimeConversionFactors(double Slope, double Offset) {
+ HostToDeviceSlope = Slope;
+ HostToDeviceOffset = Offset;
+ setTimeConversionFactorsImpl(HostToDeviceSlope, HostToDeviceOffset);
+ }
+
+ /// Hook that is called when the plugin is initialized.
+ virtual void handleInit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+ /// Hook that is called when the plugin is de-initialized.
+ virtual void handleDeinit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+ /// Hook that is called when the device image is loaded.
+ virtual void handleLoadBinary(GenericDeviceTy *Device,
+ GenericPluginTy *Plugin,
+ const StringRef InputTgtImage) {}
+
+ /// Hook that is called when memory is allocated on the device.
+ virtual void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos,
+ void *HostPtr, uint64_t Size, void *Data) {}
+
+ /// Hook that is called when memory is freed on the device.
+ virtual void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos,
+ void *TgtPtr, void *Data) {}
+
+ /// Hook that is called before launching a kernel.
+ virtual void handlePreKernelLaunch(GenericDeviceTy *Device,
+ uint32_t NumBlocks[3],
+ __tgt_async_info *AI) {}
+
+ /// Hook that is called when the kernel is finished to extract the specific
+ /// timing info for that kernel execution.
+ virtual void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+ void *Data) {}
+
+ /// Hook that is called when a data transfer happens to extract timing info
+ /// for that transfer.
+ virtual void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+ void *Data) {}
+
+ /// Allow factors for time conversion between host and device.
+ virtual void setTimeConversionFactorsImpl(double Slope, double Offset) {}
+
+ /// RAII style timer that measures the elapsed time between construction and
+ /// destruction, then invokes a callback with the profiler, start/end times,
+ /// and any captured arguments.
+ template <typename FnT, typename... ArgsT> class ProfTimerTy {
+ public:
+ ProfTimerTy(FnT &&F, GenericProfilerTy *P, GenericDeviceTy *D, ArgsT... As)
+ : Fun(F), Prof(P), Dev(D), Args(As...) {
+ assert(Prof && "GenericProfilerTy is null");
+ assert(Dev && "GenericDeviceTy is null");
+ if (Prof)
+ StartTime = Prof->getDeviceTimeStamp(Dev);
+ }
+
+ ~ProfTimerTy() {
+ assert(Prof && "GenericProfilerTy is null");
+ assert(Dev && "GenericDeviceTy is null");
+ if (Prof) {
+ uint64_t EndTime = Prof->getDeviceTimeStamp(Dev);
+ callViaUnpack(Fun, Prof, StartTime, EndTime, Args);
+ }
+ }
+
+ private:
+ FnT Fun;
+ GenericProfilerTy *Prof;
+ GenericDeviceTy *Dev;
+ uint64_t StartTime = 0;
+ std::tuple<ArgsT...> Args;
+ };
+
+ template <typename FnT, typename... ArgsT>
+ [[maybe_unused]]
+ ProfTimerTy(FnT &&, GenericProfilerTy *, ArgsT...)
+ -> ProfTimerTy<FnT, ArgsT...>;
+
+ template <typename FnT, typename... ArgsT> friend class ProfTimerTy;
+
+ /// Returns an RAII style timer, which will handle data allocation timing.
+ [[nodiscard]] auto getScopedDataAllocTimer(GenericDeviceTy *Dev,
+ void *HostPtr, uint64_t Size,
+ void *ProfData = nullptr) {
+ return ProfTimerTy(
+ [](GenericProfilerTy *P, auto... args) {
+ assert(P && "P was null");
+ P->handleDataAlloc(args...);
+ },
+ this, Dev, HostPtr, Size, ProfData);
+ }
+
+ /// Returns an RAII style timer, which will handle data deletion timing.
+ [[nodiscard]] auto getScopedDataDeleteTimer(GenericDeviceTy *Dev,
+ void *TgtPtr,
+ void *ProfData = nullptr) {
+ return ProfTimerTy(
+ [](GenericProfilerTy *P, auto... args) {
+ assert(P && "P was null");
+ P->handleDataDelete(args...);
+ },
+ this, Dev, TgtPtr, ProfData);
+ }
+
+protected:
+ /// Factors to convert host time to device time. The default values are
+ /// Computed in a way Slope * Time + Offset (inspired by Score-P).
+ double HostToDeviceSlope = 1.0;
+ double HostToDeviceOffset = .0;
+
+private:
+ /// Vendor-specific implementation to obtain device time.
+ uint64_t getDeviceTimeStamp(GenericDeviceTy *D);
+};
+
+/// Null-object used to normalize an absent profiler. Callers that do not have a
+/// profiler pass none, and the receiving method resolves to this instance, so
+/// profiling logic never has to test for a null pointer.
+GenericProfilerTy &getNoOpProfiler();
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index bc3b6db0a1ec11..b78135d77df6e0 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -29,6 +29,7 @@
#include "Shared/Requirements.h"
#include "Shared/Utils.h"
+#include "GenericProfiler.h"
#include "GlobalHandler.h"
#include "JIT.h"
#include "MemoryManager.h"
@@ -493,7 +494,8 @@ struct GenericKernelTy {
/// of it reserved for the kernel launch environment (dyn_ptr); the caller
/// owns the storage it points into.
Error launch(GenericDeviceTy &GenericDevice, KernelLaunchArgsTy &LaunchArgs,
- AsyncInfoWrapperTy &AsyncInfoWrapper) const;
+ AsyncInfoWrapperTy &AsyncInfoWrapper,
+ GenericProfilerTy *ProfilerPtr = nullptr) const;
virtual Error launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
uint32_t DynBlockMemSize,
@@ -939,17 +941,17 @@ struct PluginContextTy {
/// unused.
virtual llvm::Expected<void *> allocate(GenericDeviceTy &Device, int64_t Size,
void *HostPtr, TargetAllocTy Kind,
- size_t Alignment);
+ size_t Alignment, GenericProfilerTy *ProfilerPtr = nullptr);
/// Free a pointer returned by allocate; resolves owner/kind via
/// getAllocInfo. Requires a non-empty device set, so this is only valid on
/// user-created contexts (not on the per-plugin default context, which
/// carries no devices).
- virtual llvm::Error deallocate(void *Ptr);
+ virtual llvm::Error deallocate(void *Ptr, GenericProfilerTy *ProfilerPtr = nullptr);
/// Free a pointer when the caller already knows the owning device and kind.
virtual llvm::Error deallocate(GenericDeviceTy &Device, void *Ptr,
- TargetAllocTy Kind);
+ TargetAllocTy Kind, GenericProfilerTy *ProfilerPtr = nullptr);
/// Look up the allocation containing Ptr. Returns NOT_FOUND when Ptr is not
/// known to this context. Only valid on user-created contexts.
@@ -1042,20 +1044,23 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
/// Initialize the device. After this call, the device should be already
/// working and ready to accept queries or modifications.
- Error init(GenericPluginTy &Plugin);
- virtual Error initImpl(GenericPluginTy &Plugin) = 0;
+ Error init(GenericPluginTy &Plugin, GenericProfilerTy *ProfilerPtr = nullptr);
+ virtual Error initImpl(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) = 0;
/// Deinitialize the device and free all its resources. After this call, the
/// device is no longer considered ready, so no queries or modifications are
/// allowed.
- Error deinit(GenericPluginTy &Plugin);
+ Error deinit(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr = nullptr);
virtual Error deinitImpl() = 0;
/// Load the binary image into the device and return the target table. When
/// \p Context is null the plugin's driver-scoped default context is used.
- Expected<DeviceImageTy *> loadBinary(GenericPluginTy &Plugin,
- StringRef TgtImage,
- PluginContextTy *Context);
+ Expected<DeviceImageTy *>
+ loadBinary(GenericPluginTy &Plugin, StringRef TgtImage,
+ PluginContextTy *Context,
+ GenericProfilerTy *ProfilerPtr = nullptr);
virtual Expected<DeviceImageTy *>
loadBinaryImpl(std::unique_ptr<MemoryBuffer> &&TgtImage, int32_t ImageId,
PluginContextTy *Context) = 0;
@@ -1115,10 +1120,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
/// Allocate data on the device or involving the device.
Expected<void *> dataAlloc(int64_t Size, void *HostPtr, TargetAllocTy Kind,
- size_t Alignment);
+ size_t Alignment,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Deallocate data from the device or involving the device.
- Error dataDelete(void *TgtPtr, TargetAllocTy Kind);
+ Error dataDelete(void *TgtPtr, TargetAllocTy Kind,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Pin or register host memory to optimize transfers and return the device
/// accessible pointer that devices should use for memory transfers involving
@@ -1222,7 +1229,8 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
/// Run the kernel associated with \p EntryPtr
Error launchKernel(void *EntryPtr, KernelLaunchArgsTy &LaunchArgs,
- __tgt_async_info *AsyncInfo);
+ __tgt_async_info *AsyncInfo,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Enqueue a host call to AsyncInfo
Error enqueueHostCall(void (*Callback)(void *), void *UserData,
@@ -1301,6 +1309,11 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
uint32_t getDebugKind() const { return OMPX_DebugKind; }
virtual uint64_t getClockFrequency() const { return CLOCKS_PER_SEC; }
+ /// Get a device-specific timestamp in nanoseconds, used by the profiler
+ /// for timing device operations. Subclasses should override this to provide
+ /// hardware-accurate timestamps (e.g., via HSA system info).
+ virtual uint64_t getDeviceTimeStamp() { return 0; }
+
/// Get target compute unit kind (e.g., sm_80, or gfx908).
virtual std::string getComputeUnitKind() const { return "unknown"; }
@@ -1588,7 +1601,7 @@ struct GenericPluginTy {
virtual Expected<int32_t> initImpl() = 0;
/// Deinitialize the plugin and release the resources.
- Error deinit();
+ Error deinit(GenericProfilerTy *ProfilerPtr = nullptr);
virtual Error deinitImpl() = 0;
/// Create a new device for the underlying plugin.
@@ -1668,12 +1681,12 @@ struct GenericPluginTy {
/// Tear down any target-specific doorbell resources.
virtual Error deinitRPCDoorbell() { return Plugin::success(); }
- /// Get a reference to the record and replay interface for the plugin.
/// Initialize a device within the plugin.
- Error initDevice(int32_t DeviceId);
+ Error initDevice(int32_t DeviceId, GenericProfilerTy *ProfilerPtr = nullptr);
/// Deinitialize a device within the plugin and release its resources.
- Error deinitDevice(int32_t DeviceId);
+ Error deinitDevice(int32_t DeviceId,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Indicate whether data can be exchanged directly between two devices under
/// this same plugin. If this function returns true, it's safe to call the
@@ -1765,7 +1778,8 @@ struct GenericPluginTy {
int32_t is_device_initialized(int32_t DeviceId) const;
/// Initialize the device inside of the plugin.
- int32_t init_device(int32_t DeviceId);
+ int32_t init_device(int32_t DeviceId,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Return the number of devices this plugin can support.
int32_t number_of_devices();
@@ -1782,13 +1796,16 @@ struct GenericPluginTy {
/// Loads the associated binary into the plugin and returns a handle to it.
int32_t load_binary(int32_t DeviceId, __tgt_device_image *TgtImage,
- __tgt_device_binary *Binary);
+ __tgt_device_binary *Binary,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Allocates memory that is accessively to the given device.
- void *data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, int32_t Kind);
+ void *data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, int32_t Kind,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Deallocates memory on the given device.
- int32_t data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind);
+ int32_t data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Locks / pins host memory using the plugin runtime.
int32_t data_lock(int32_t DeviceId, void *Ptr, int64_t Size,
@@ -1835,7 +1852,8 @@ struct GenericPluginTy {
/// Begin executing a kernel on the given device.
int32_t launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
KernelLaunchArgsTy &LaunchArgs,
- __tgt_async_info *AsyncInfoPtr);
+ __tgt_async_info *AsyncInfoPtr,
+ GenericProfilerTy *ProfilerPtr = nullptr);
/// Synchronize an asyncrhonous queue with the plugin runtime.
int32_t synchronize(int32_t DeviceId, __tgt_async_info *AsyncInfoPtr);
diff --git a/offload/plugins-nextgen/common/src/GenericProfiler.cpp b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
new file mode 100644
index 00000000000000..5486c40a7a9d2f
--- /dev/null
+++ b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
@@ -0,0 +1,36 @@
+//===- GenericProfiler.cpp - GenericProfiler implementation ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "GenericProfiler.h"
+#include "PluginInterface.h"
+
+#include <cstdint>
+#include <memory>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+uint64_t GenericProfilerTy::getDeviceTimeStamp(GenericDeviceTy *D) {
+ if (!D)
+ return 0;
+
+ return D->getDeviceTimeStamp();
+}
+
+GenericProfilerTy &getNoOpProfiler() {
+ static GenericProfilerTy NoOpProfiler;
+ return NoOpProfiler;
+}
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 61b8b5fe1155b4..dc75d75a8507e0 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -26,6 +26,8 @@
#include "omp-tools.h"
#endif
+#include "GenericProfiler.h"
+
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/Support/Error.h"
@@ -47,7 +49,9 @@ using namespace llvm::offload::debug;
AsyncInfoWrapperTy::AsyncInfoWrapperTy(GenericDeviceTy &Device,
__tgt_async_info *AsyncInfoPtr)
: Device(Device),
- AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {}
+ AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {
+ LocalAsyncInfo.ProfilerData = nullptr;
+}
Error AsyncInfoWrapperTy::synchronize() {
assert(AsyncInfoPtr && "AsyncInfoWrapperTy already finalized");
@@ -170,9 +174,15 @@ GenericKernelTy::getKernelLaunchEnvironment(
DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
sizeof(KernelLaunchEnvironmentTy));
+ // Temporarily suppress ProfilerData so the KLE upload is not traced as
+ // a user data operation.
+ __tgt_async_info *AI = AsyncInfoWrapper;
+ void *SavedProfilerData = AI->ProfilerData;
+ AI->ProfilerData = nullptr;
auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
sizeof(KernelLaunchEnvironmentTy),
AsyncInfoWrapper);
+ AI->ProfilerData = SavedProfilerData;
if (Err)
return Err;
return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
@@ -246,7 +256,9 @@ GenericKernelTy::prepareBlockMemory(GenericDeviceTy &GenericDevice,
Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice,
KernelLaunchArgsTy &LaunchArgs,
- AsyncInfoWrapperTy &AsyncInfoWrapper) const {
+ AsyncInfoWrapperTy &AsyncInfoWrapper,
+ GenericProfilerTy *ProfilerPtr) const {
+ GenericProfilerTy &Profiler = ProfilerPtr ? *ProfilerPtr : getNoOpProfiler();
uint32_t EffectiveNumThreads[3] = {LaunchArgs.UserThreadLimit[0],
LaunchArgs.UserThreadLimit[1],
LaunchArgs.UserThreadLimit[2]};
@@ -324,6 +336,9 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice,
RRHandle = *RRHandleOrErr;
}
+ Profiler.handlePreKernelLaunch(&GenericDevice, EffectiveNumBlocks,
+ AsyncInfoWrapper);
+
if (auto Err =
launchImpl(GenericDevice, EffectiveNumThreads, EffectiveNumBlocks,
DynBlockMemConf.NativeSize, LaunchArgs, AsyncInfoWrapper))
@@ -509,8 +524,11 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
}
}
-Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
- if (auto Err = initImpl(Plugin))
+Error GenericDeviceTy::init(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) {
+ GenericProfilerTy &Profiler = ProfilerPtr ? *ProfilerPtr : getNoOpProfiler();
+
+ if (auto Err = initImpl(Plugin, ProfilerPtr))
return Err;
#ifdef OMPT_SUPPORT
@@ -525,6 +543,8 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
}
#endif
+ Profiler.handleInit(this, &Plugin);
+
// Read and reinitialize the envars that depend on the device initialization.
// Notice these two envars may change the stack size and heap size of the
// device, so they need the device properly initialized.
@@ -579,7 +599,10 @@ Error GenericDeviceTy::unloadBinary(DeviceImageTy *Image) {
return unloadBinaryImpl(Image);
}
-Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
+Error GenericDeviceTy::deinit(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) {
+ GenericProfilerTy &Profiler = ProfilerPtr ? *ProfilerPtr : getNoOpProfiler();
+
// Run the global destructors first in case they required the RPC server.
for (auto &I : LoadedImages) {
if (auto Err = callGlobalDestructors(Plugin, *I))
@@ -612,11 +635,16 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
}
#endif
+ Profiler.handleDeinit(this, &Plugin);
+
return deinitImpl();
}
Expected<DeviceImageTy *>
GenericDeviceTy::loadBinary(GenericPluginTy &Plugin, StringRef InputTgtImage,
- PluginContextTy *Context) {
+ PluginContextTy *Context,
+ GenericProfilerTy *ProfilerPtr) {
+ GenericProfilerTy &Profiler = ProfilerPtr ? *ProfilerPtr : getNoOpProfiler();
+
ODBG(OLDT_Init) << "Load data from image "
<< static_cast<const void *>(InputTgtImage.bytes_begin());
@@ -671,6 +699,8 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin, StringRef InputTgtImage,
}
#endif
+ Profiler.handleLoadBinary(this, &Plugin, InputTgtImage);
+
// Call any global constructors present on the device.
if (auto Err = callGlobalConstructors(Plugin, *Image))
return std::move(Err);
@@ -978,7 +1008,12 @@ Error GenericDeviceTy::getDeviceMemorySize(uint64_t &DSize) {
Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
TargetAllocTy Kind,
- size_t Alignment) {
+ size_t Alignment,
+ GenericProfilerTy *ProfilerPtr) {
+ GenericProfilerTy &Profiler = ProfilerPtr ? *ProfilerPtr : getNoOpProfiler();
+
+ auto ProfTimer = Profiler.getScopedDataAllocTimer(this, HostPtr, Size);
+
void *Alloc = nullptr;
// TODO Check alignment.
@@ -1030,7 +1065,12 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
return Alloc;
}
-Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
+Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind,
+ GenericProfilerTy *ProfilerPtr) {
+ GenericProfilerTy &Profiler = ProfilerPtr ? *ProfilerPtr : getNoOpProfiler();
+
+ auto ProfTimer = Profiler.getScopedDataDeleteTimer(this, TgtPtr);
+
// Free is a noop when recording or replaying.
if (RecordReplay && RecordReplay->isRecordingOrReplaying())
return RecordReplay->deallocate(TgtPtr);
@@ -1131,7 +1171,8 @@ Error GenericDeviceTy::dataPrefetch(size_t Count, const void **Mems,
Error GenericDeviceTy::launchKernel(void *EntryPtr,
KernelLaunchArgsTy &LaunchArgs,
- __tgt_async_info *AsyncInfo) {
+ __tgt_async_info *AsyncInfo,
+ GenericProfilerTy *ProfilerPtr) {
AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
GenericKernelTy &GenericKernel =
@@ -1149,7 +1190,8 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr,
.emplace(&GenericKernel, std::move(StackTrace), AsyncInfo);
}
- auto Err = GenericKernel.launch(*this, LaunchArgs, AsyncInfoWrapper);
+ auto Err =
+ GenericKernel.launch(*this, LaunchArgs, AsyncInfoWrapper, ProfilerPtr);
AsyncInfoWrapper.finalize(Err);
@@ -1225,22 +1267,22 @@ MemoryManagerTy *PluginContextTy::getHostMemoryManager() {
Expected<void *> PluginContextTy::allocate(GenericDeviceTy &Device,
int64_t Size, void *HostPtr,
TargetAllocTy Kind,
- size_t Alignment) {
+ size_t Alignment, GenericProfilerTy *ProfilerPtr) {
// Record-replay hands out interior pointers into a preallocated slab so
// recorded kernels can re-execute at their original addresses; the MM pool
// must be bypassed for those allocations to reach the RR bump allocator.
if (auto *RR = Device.getRecordReplay(); RR && RR->isRecordingOrReplaying())
- return Device.dataAlloc(Size, HostPtr, Kind, Alignment);
+ return Device.dataAlloc(Size, HostPtr, Kind, Alignment, ProfilerPtr);
MemoryManagerTy *MM = (Kind == TARGET_ALLOC_HOST)
? getHostMemoryManager()
: getDeviceMemoryManagerFor(Device, Kind);
if (MM)
return MM->allocate(Size, HostPtr, Alignment);
- return Device.dataAlloc(Size, HostPtr, Kind, Alignment);
+ return Device.dataAlloc(Size, HostPtr, Kind, Alignment, ProfilerPtr);
}
-Error PluginContextTy::deallocate(void *Ptr) {
+Error PluginContextTy::deallocate(void *Ptr, GenericProfilerTy *ProfilerPtr) {
assert(!Devices.empty() && "context constructed without devices");
auto InfoOrErr = getAllocInfo(Ptr);
if (!InfoOrErr)
@@ -1248,22 +1290,22 @@ Error PluginContextTy::deallocate(void *Ptr) {
GenericDeviceTy *OwnerDevice = InfoOrErr->Device;
if (!OwnerDevice)
OwnerDevice = Devices.front();
- return deallocate(*OwnerDevice, Ptr, InfoOrErr->Kind);
+ return deallocate(*OwnerDevice, Ptr, InfoOrErr->Kind, ProfilerPtr);
}
Error PluginContextTy::deallocate(GenericDeviceTy &Device, void *Ptr,
- TargetAllocTy Kind) {
+ TargetAllocTy Kind, GenericProfilerTy *ProfilerPtr) {
// Symmetric with allocate: record-replay allocations never entered the MM
// pool, so route their free through dataDelete's RR shortcut.
if (auto *RR = Device.getRecordReplay(); RR && RR->isRecordingOrReplaying())
- return Device.dataDelete(Ptr, Kind);
+ return Device.dataDelete(Ptr, Kind, ProfilerPtr);
MemoryManagerTy *MM = (Kind == TARGET_ALLOC_HOST)
? getHostMemoryManager()
: getDeviceMemoryManagerFor(Device, Kind);
if (MM)
return MM->free(Ptr);
- return Device.dataDelete(Ptr, Kind);
+ return Device.dataDelete(Ptr, Kind, ProfilerPtr);
}
PluginContextTy &
@@ -1412,7 +1454,7 @@ Error GenericPluginTy::init() {
return Plugin::success();
}
-Error GenericPluginTy::deinit() {
+Error GenericPluginTy::deinit(GenericProfilerTy *ProfilerPtr) {
assert(Initialized && "Plugin was not initialized!");
// Release context-held resources before the devices that back them.
@@ -1421,7 +1463,7 @@ Error GenericPluginTy::deinit() {
// Deinitialize all active devices.
for (int32_t DeviceId = 0; DeviceId < NumDevices; ++DeviceId) {
if (Devices[DeviceId]) {
- if (auto Err = deinitDevice(DeviceId))
+ if (auto Err = deinitDevice(DeviceId, ProfilerPtr))
return Err;
}
assert(!Devices[DeviceId] && "Device was not deinitialized");
@@ -1445,7 +1487,8 @@ Error GenericPluginTy::deinit() {
return Plugin::success();
}
-Error GenericPluginTy::initDevice(int32_t DeviceId) {
+Error GenericPluginTy::initDevice(int32_t DeviceId,
+ GenericProfilerTy *ProfilerPtr) {
assert(!Devices[DeviceId] && "Device already initialized");
// Create the device and save the reference.
@@ -1456,16 +1499,17 @@ Error GenericPluginTy::initDevice(int32_t DeviceId) {
Devices[DeviceId] = Device;
// Initialize the device and its resources.
- return Device->init(*this);
+ return Device->init(*this, ProfilerPtr);
}
-Error GenericPluginTy::deinitDevice(int32_t DeviceId) {
+Error GenericPluginTy::deinitDevice(int32_t DeviceId,
+ GenericProfilerTy *ProfilerPtr) {
// The device may be already deinitialized.
if (Devices[DeviceId] == nullptr)
return Plugin::success();
// Deinitialize the device and release its resources.
- if (auto Err = Devices[DeviceId]->deinit(*this))
+ if (auto Err = Devices[DeviceId]->deinit(*this, ProfilerPtr))
return Err;
// Delete the device and invalidate its reference.
@@ -1581,8 +1625,9 @@ int32_t GenericPluginTy::is_device_initialized(int32_t DeviceId) const {
return isValidDeviceId(DeviceId) && Devices[DeviceId] != nullptr;
}
-int32_t GenericPluginTy::init_device(int32_t DeviceId) {
- auto Err = initDevice(DeviceId);
+int32_t GenericPluginTy::init_device(int32_t DeviceId,
+ GenericProfilerTy *ProfilerPtr) {
+ auto Err = initDevice(DeviceId, ProfilerPtr);
if (Err) {
REPORT() << "Failure to initialize device " << DeviceId << ": "
<< toString(std::move(Err));
@@ -1618,12 +1663,14 @@ int32_t GenericPluginTy::initialize_record_replay(
int32_t GenericPluginTy::load_binary(int32_t DeviceId,
__tgt_device_image *TgtImage,
- __tgt_device_binary *Binary) {
+ __tgt_device_binary *Binary,
+ GenericProfilerTy *ProfilerPtr) {
GenericDeviceTy &Device = getDevice(DeviceId);
StringRef Buffer(reinterpret_cast<const char *>(TgtImage->ImageStart),
utils::getPtrDiff(TgtImage->ImageEnd, TgtImage->ImageStart));
- auto ImageOrErr = Device.loadBinary(*this, Buffer, /*Context=*/nullptr);
+ auto ImageOrErr =
+ Device.loadBinary(*this, Buffer, /*Context=*/nullptr, ProfilerPtr);
if (!ImageOrErr) {
auto Err = ImageOrErr.takeError();
REPORT() << "Failure to load binary image " << TgtImage << " on device "
@@ -1640,11 +1687,12 @@ int32_t GenericPluginTy::load_binary(int32_t DeviceId,
}
void *GenericPluginTy::data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
- int32_t Kind) {
+ int32_t Kind, GenericProfilerTy *ProfilerPtr) {
auto &Device = getDevice(DeviceId);
auto AllocOrErr = getDefaultContext(Device).allocate(
Device, Size, HostPtr, static_cast<TargetAllocTy>(Kind),
- /*Alignment=*/0);
+ /*Alignment=*/0, ProfilerPtr);
+
if (!AllocOrErr) {
REPORT() << "Failure to allocate device memory: "
<< toString(AllocOrErr.takeError());
@@ -1655,10 +1703,10 @@ void *GenericPluginTy::data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
}
int32_t GenericPluginTy::data_delete(int32_t DeviceId, void *TgtPtr,
- int32_t Kind) {
+ int32_t Kind, GenericProfilerTy *ProfilerPtr) {
auto &Device = getDevice(DeviceId);
if (auto Err = getDefaultContext(Device).deallocate(
- Device, TgtPtr, static_cast<TargetAllocTy>(Kind))) {
+ Device, TgtPtr, static_cast<TargetAllocTy>(Kind), ProfilerPtr)) {
REPORT() << "Failure to deallocate device pointer " << TgtPtr << ": "
<< toString(std::move(Err));
return OFFLOAD_FAIL;
@@ -1788,9 +1836,10 @@ int32_t GenericPluginTy::data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
int32_t GenericPluginTy::launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
KernelLaunchArgsTy &LaunchArgs,
- __tgt_async_info *AsyncInfoPtr) {
- auto Err =
- getDevice(DeviceId).launchKernel(TgtEntryPtr, LaunchArgs, AsyncInfoPtr);
+ __tgt_async_info *AsyncInfoPtr,
+ GenericProfilerTy *ProfilerPtr) {
+ auto Err = getDevice(DeviceId).launchKernel(TgtEntryPtr, LaunchArgs,
+ AsyncInfoPtr, ProfilerPtr);
if (Err) {
REPORT() << "Failure to run target region " << TgtEntryPtr << " in device "
<< DeviceId << ": " << toString(std::move(Err));
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 2dc63a2bea8d38..198588aa2023cf 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -279,7 +279,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
~CUDADeviceTy() {}
/// Initialize the device, its resources and get its properties.
- Error initImpl(GenericPluginTy &Plugin) override {
+ Error initImpl(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) override {
CUresult Res = cuDeviceGet(&Device, DeviceId);
if (auto Err = Plugin::check(Res, "error in cuDeviceGet: %s"))
return Err;
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index a594cf0fbc6ed2..55264eed1aced2 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -147,7 +147,10 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
~GenELF64DeviceTy() {}
/// Initialize the device, which is a no-op
- Error initImpl(GenericPluginTy &Plugin) override { return Plugin::success(); }
+ Error initImpl(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) override {
+ return Plugin::success();
+ }
/// Unload the binary image
///
@@ -493,9 +496,9 @@ struct GenELF64PluginContextTy final : public PluginContextTy {
Expected<void *> allocate(GenericDeviceTy &Device, int64_t Size,
void *HostPtr, TargetAllocTy Kind,
- size_t Alignment) override {
+ size_t Alignment, GenericProfilerTy *ProfilerPtr) override {
auto PtrOrErr =
- PluginContextTy::allocate(Device, Size, HostPtr, Kind, Alignment);
+ PluginContextTy::allocate(Device, Size, HostPtr, Kind, Alignment, ProfilerPtr);
if (!PtrOrErr)
return PtrOrErr.takeError();
void *Ptr = *PtrOrErr;
@@ -505,12 +508,12 @@ struct GenELF64PluginContextTy final : public PluginContextTy {
}
Error deallocate(GenericDeviceTy &Device, void *Ptr,
- TargetAllocTy Kind) override {
+ TargetAllocTy Kind, GenericProfilerTy *ProfilerPtr) override {
{
std::lock_guard<std::mutex> Lock(AllocsMutex);
Allocs.erase(Ptr);
}
- return PluginContextTy::deallocate(Device, Ptr, Kind);
+ return PluginContextTy::deallocate(Device, Ptr, Kind, ProfilerPtr);
}
Expected<PluginAllocInfoTy> getAllocInfo(const void *Ptr) override {
diff --git a/offload/plugins-nextgen/level_zero/include/L0Device.h b/offload/plugins-nextgen/level_zero/include/L0Device.h
index 3341f261deaf95..aa102af83ff342 100644
--- a/offload/plugins-nextgen/level_zero/include/L0Device.h
+++ b/offload/plugins-nextgen/level_zero/include/L0Device.h
@@ -181,7 +181,8 @@ class L0DeviceTy final : public GenericDeviceTy {
}
Error setContext() override { return Plugin::success(); }
- Error initImpl(GenericPluginTy &Plugin) override;
+ Error initImpl(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) override;
Error deinitImpl() override;
ze_device_handle_t getZeDevice() const { return zeDevice; }
diff --git a/offload/plugins-nextgen/level_zero/src/L0Device.cpp b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
index e41268d7eb843a..c539e7a23e5235 100644
--- a/offload/plugins-nextgen/level_zero/src/L0Device.cpp
+++ b/offload/plugins-nextgen/level_zero/src/L0Device.cpp
@@ -159,7 +159,8 @@ void L0DeviceTy::reportDeviceInfo() const {
});
}
-Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
+Error L0DeviceTy::initImpl(GenericPluginTy &Plugin,
+ GenericProfilerTy *ProfilerPtr) {
const auto &Options = getPlugin().getOptions();
uint32_t Count = 1;
More information about the llvm-commits
mailing list