[llvm-branch-commits] [llvm] [offload] Copy the kernel launch environment with a direct-write memcpy (PR #231923)
Robert Imschweiler via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 29 08:00:40 PDT 2026
https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/231923
>From 4f77a39395a167a6790397b1e080ea8b61ee9e43 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 14 Oct 2026 07:12:35 -0500
Subject: [PATCH 1/3] [offload] Copy the kernel launch environment with a
direct-write memcpy
Claude assisted with this patch.
---
offload/libompaccsupport/device.cpp | 20 +++--
.../offloading/kernel_launch_environment.c | 76 +++++++++++++++++++
2 files changed, 91 insertions(+), 5 deletions(-)
create mode 100644 offload/test/offloading/kernel_launch_environment.c
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index e62a34a7a5991b..22919c07027cde 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -772,12 +772,22 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion,
freeAfterSynchronization(Device, AsyncInfo, *ReductionAllocOrErr);
}
- if (Device.submitData(*AllocOrErr, &LocalKLE,
- sizeof(KernelLaunchEnvironmentTy), AsyncInfo,
- /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr,
- /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv"))
+ INFO(OMP_INFOTYPE_DATA_TRANSFER, Device.DeviceID,
+ "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
+ ", Size=%" PRId64 ", Name=KernelLaunchEnv\n",
+ DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
+ sizeof(KernelLaunchEnvironmentTy));
+
+ // The allocation is fresh, so no work pending on the queue accesses it and
+ // the copy can bypass the queue. Without a queue, the plugin may write the
+ // memory directly.
+ if (const auto *Res = olMemcpy(
+ /*Queue=*/nullptr, *AllocOrErr, Device.DeviceHandle, &LocalKLE,
+ /*SrcDevice=*/nullptr, sizeof(KernelLaunchEnvironmentTy)))
return createError(ErrorCode::BackendFailure,
- "failed to copy the kernel launch environment");
+ "failed to copy the kernel launch environment to "
+ "device %d: %s",
+ Device.DeviceID, Res->Details);
return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
}
diff --git a/offload/test/offloading/kernel_launch_environment.c b/offload/test/offloading/kernel_launch_environment.c
new file mode 100644
index 00000000000000..c6703f3b2b3805
--- /dev/null
+++ b/offload/test/offloading/kernel_launch_environment.c
@@ -0,0 +1,76 @@
+// Stress the kernel launch environment (KLE) of concurrent launches.
+//
+// A cross-team reduction gives the launch a KLE, which carries that launch's
+// own reduction buffer. The KLE is copied to the device synchronously, which the
+// plugin may do by storing it in place where device memory is host-writable.
+// Either has to hold up when launches overlap: every launch needs its own KLE,
+// and a launch's KLE has to be visible to the device before its kernel runs
+// and stay intact until it finishes. Otherwise a launch reduces into another
+// launch's buffer, or reads a KLE that has not landed yet. Every launch here
+// accumulates a distinct value, so either shows up as a wrong sum.
+//
+// RUN: %libomptarget-compile-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+// RUN: rm -rf %t.rr && mkdir -p %t.rr
+// RUN: env LIBOMPTARGET_RECORD=1 LIBOMPTARGET_RECORD_DIR=%t.rr \
+// RUN: %libomptarget-run-generic
+//
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define NUM_LAUNCHES 32
+#define N 4096
+
+// Launch k accumulates k + 1 per element.
+static long expected(int k) { return (long)(k + 1) * N; }
+
+static int check(const char *Phase, long *Results) {
+ int Errors = 0;
+ for (int k = 0; k < NUM_LAUNCHES; k++)
+ if (Results[k] != expected(k)) {
+ fprintf(stderr, "%s: launch %d reduced to %ld, expected %ld\n", Phase, k,
+ Results[k], expected(k));
+ Errors++;
+ }
+ return Errors;
+}
+
+int main(void) {
+ static long Results[NUM_LAUNCHES];
+ int Errors = 0;
+
+ // Launches issued back to back without an intervening synchronization, so
+ // that several launches are in flight at once.
+ for (int k = 0; k < NUM_LAUNCHES; k++) {
+ Results[k] = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Results[k : 1]) \
+ reduction(+ : Results[k]) firstprivate(k) nowait
+ for (int i = 0; i < N; i++)
+ Results[k] += k + 1;
+ }
+#pragma omp taskwait
+ Errors += check("nowait", Results);
+
+ // Same, but with the launches and the synchronizations spread over several
+ // host threads: a thread finalizing its queue must not disturb the launch
+ // environment of a launch another thread still has in flight.
+#pragma omp parallel for num_threads(8)
+ for (int k = 0; k < NUM_LAUNCHES; k++) {
+ long Sum = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Sum) \
+ reduction(+ : Sum) firstprivate(k)
+ for (int i = 0; i < N; i++)
+ Sum += k + 1;
+ Results[k] = Sum;
+ }
+ Errors += check("threaded", Results);
+
+ if (Errors)
+ return 1;
+ printf("PASS\n");
+ return 0;
+}
>From 460b70745b1ca17c7946dbf68c426ade3256af25 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 28 Oct 2026 16:00:41 -0500
Subject: [PATCH 2/3] use submitDataSync
---
offload/libompaccsupport/device.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 22919c07027cde..5698340083c756 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -781,13 +781,12 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion,
// The allocation is fresh, so no work pending on the queue accesses it and
// the copy can bypass the queue. Without a queue, the plugin may write the
// memory directly.
- if (const auto *Res = olMemcpy(
- /*Queue=*/nullptr, *AllocOrErr, Device.DeviceHandle, &LocalKLE,
- /*SrcDevice=*/nullptr, sizeof(KernelLaunchEnvironmentTy)))
+ if (Device.submitDataSync(*AllocOrErr, &LocalKLE,
+ sizeof(KernelLaunchEnvironmentTy)))
return createError(ErrorCode::BackendFailure,
"failed to copy the kernel launch environment to "
- "device %d: %s",
- Device.DeviceID, Res->Details);
+ "device %d",
+ Device.DeviceID);
return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
}
>From 1fb2c981da1d9370b31bfe6278249fea5b22594d Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 29 Oct 2026 09:56:58 -0500
Subject: [PATCH 3/3] only use sync copy on AMDGPU; improve logging
---
offload/libompaccsupport/device.cpp | 23 +++++++++++++----------
1 file changed, 13 insertions(+), 10 deletions(-)
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 5698340083c756..fae218efe23a59 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -772,17 +772,20 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion,
freeAfterSynchronization(Device, AsyncInfo, *ReductionAllocOrErr);
}
- INFO(OMP_INFOTYPE_DATA_TRANSFER, Device.DeviceID,
- "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
- ", Size=%" PRId64 ", Name=KernelLaunchEnv\n",
- DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
- sizeof(KernelLaunchEnvironmentTy));
-
// The allocation is fresh, so no work pending on the queue accesses it and
- // the copy can bypass the queue. Without a queue, the plugin may write the
- // memory directly.
- if (Device.submitDataSync(*AllocOrErr, &LocalKLE,
- sizeof(KernelLaunchEnvironmentTy)))
+ // the copy can bypass the queue. Without a queue, the AMDGPU plugin may write
+ // the memory directly. Other backends copy on the queue.
+ int32_t Ret =
+ Device.getBackend() == OL_PLATFORM_BACKEND_AMDGPU
+ ? Device.submitDataSync(*AllocOrErr, &LocalKLE,
+ sizeof(KernelLaunchEnvironmentTy),
+ /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr,
+ /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv")
+ : Device.submitData(*AllocOrErr, &LocalKLE,
+ sizeof(KernelLaunchEnvironmentTy), AsyncInfo,
+ /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr,
+ /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv");
+ if (Ret)
return createError(ErrorCode::BackendFailure,
"failed to copy the kernel launch environment to "
"device %d",
More information about the llvm-branch-commits
mailing list