[llvm-branch-commits] [llvm] [offload] Copy the kernel launch environment with a direct-write memcpy (PR #231923)

Robert Imschweiler via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Fri Oct 30 04:13:45 PDT 2026


https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/231923

>From d8d5863582454c5216a67ef38a7d0cad06503d8b Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 14 Oct 2026 07:12:35 -0500
Subject: [PATCH 1/3] [offload] Copy the kernel launch environment with a
 direct-write memcpy

Claude assisted with this patch.
---
 offload/libompaccsupport/device.cpp           | 20 +++--
 .../offloading/kernel_launch_environment.c    | 76 +++++++++++++++++++
 2 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 offload/test/offloading/kernel_launch_environment.c

diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index e62a34a7a5991b..22919c07027cde 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -772,12 +772,22 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion,
     freeAfterSynchronization(Device, AsyncInfo, *ReductionAllocOrErr);
   }
 
-  if (Device.submitData(*AllocOrErr, &LocalKLE,
-                        sizeof(KernelLaunchEnvironmentTy), AsyncInfo,
-                        /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr,
-                        /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv"))
+  INFO(OMP_INFOTYPE_DATA_TRANSFER, Device.DeviceID,
+       "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
+       ", Size=%" PRId64 ", Name=KernelLaunchEnv\n",
+       DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
+       sizeof(KernelLaunchEnvironmentTy));
+
+  // The allocation is fresh, so no work pending on the queue accesses it and
+  // the copy can bypass the queue. Without a queue, the plugin may write the
+  // memory directly.
+  if (const auto *Res = olMemcpy(
+          /*Queue=*/nullptr, *AllocOrErr, Device.DeviceHandle, &LocalKLE,
+          /*SrcDevice=*/nullptr, sizeof(KernelLaunchEnvironmentTy)))
     return createError(ErrorCode::BackendFailure,
-                       "failed to copy the kernel launch environment");
+                       "failed to copy the kernel launch environment to "
+                       "device %d: %s",
+                       Device.DeviceID, Res->Details);
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
 }
 
diff --git a/offload/test/offloading/kernel_launch_environment.c b/offload/test/offloading/kernel_launch_environment.c
new file mode 100644
index 00000000000000..c6703f3b2b3805
--- /dev/null
+++ b/offload/test/offloading/kernel_launch_environment.c
@@ -0,0 +1,76 @@
+// Stress the kernel launch environment (KLE) of concurrent launches.
+//
+// A cross-team reduction gives the launch a KLE, which carries that launch's
+// own reduction buffer. The KLE is copied to the device synchronously, which the
+// plugin may do by storing it in place where device memory is host-writable.
+// Either has to hold up when launches overlap: every launch needs its own KLE,
+// and a launch's KLE has to be visible to the device before its kernel runs
+// and stay intact until it finishes. Otherwise a launch reduces into another
+// launch's buffer, or reads a KLE that has not landed yet. Every launch here
+// accumulates a distinct value, so either shows up as a wrong sum.
+//
+// RUN: %libomptarget-compile-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+// RUN: rm -rf %t.rr && mkdir -p %t.rr
+// RUN: env LIBOMPTARGET_RECORD=1 LIBOMPTARGET_RECORD_DIR=%t.rr \
+// RUN:   %libomptarget-run-generic
+//
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define NUM_LAUNCHES 32
+#define N 4096
+
+// Launch k accumulates k + 1 per element.
+static long expected(int k) { return (long)(k + 1) * N; }
+
+static int check(const char *Phase, long *Results) {
+  int Errors = 0;
+  for (int k = 0; k < NUM_LAUNCHES; k++)
+    if (Results[k] != expected(k)) {
+      fprintf(stderr, "%s: launch %d reduced to %ld, expected %ld\n", Phase, k,
+              Results[k], expected(k));
+      Errors++;
+    }
+  return Errors;
+}
+
+int main(void) {
+  static long Results[NUM_LAUNCHES];
+  int Errors = 0;
+
+  // Launches issued back to back without an intervening synchronization, so
+  // that several launches are in flight at once.
+  for (int k = 0; k < NUM_LAUNCHES; k++) {
+    Results[k] = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Results[k : 1])  \
+    reduction(+ : Results[k]) firstprivate(k) nowait
+    for (int i = 0; i < N; i++)
+      Results[k] += k + 1;
+  }
+#pragma omp taskwait
+  Errors += check("nowait", Results);
+
+  // Same, but with the launches and the synchronizations spread over several
+  // host threads: a thread finalizing its queue must not disturb the launch
+  // environment of a launch another thread still has in flight.
+#pragma omp parallel for num_threads(8)
+  for (int k = 0; k < NUM_LAUNCHES; k++) {
+    long Sum = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Sum)             \
+    reduction(+ : Sum) firstprivate(k)
+    for (int i = 0; i < N; i++)
+      Sum += k + 1;
+    Results[k] = Sum;
+  }
+  Errors += check("threaded", Results);
+
+  if (Errors)
+    return 1;
+  printf("PASS\n");
+  return 0;
+}

>From 614235bbfa8733171ee6ac4831db87b7db3ce070 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 28 Oct 2026 16:00:41 -0500
Subject: [PATCH 2/3] use submitDataSync

---
 offload/libompaccsupport/device.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 22919c07027cde..5698340083c756 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -781,13 +781,12 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion,
   // The allocation is fresh, so no work pending on the queue accesses it and
   // the copy can bypass the queue. Without a queue, the plugin may write the
   // memory directly.
-  if (const auto *Res = olMemcpy(
-          /*Queue=*/nullptr, *AllocOrErr, Device.DeviceHandle, &LocalKLE,
-          /*SrcDevice=*/nullptr, sizeof(KernelLaunchEnvironmentTy)))
+  if (Device.submitDataSync(*AllocOrErr, &LocalKLE,
+                            sizeof(KernelLaunchEnvironmentTy)))
     return createError(ErrorCode::BackendFailure,
                        "failed to copy the kernel launch environment to "
-                       "device %d: %s",
-                       Device.DeviceID, Res->Details);
+                       "device %d",
+                       Device.DeviceID);
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
 }
 

>From 99f78a2b16264675358c4e03d589d1c7766f1985 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 29 Oct 2026 09:56:58 -0500
Subject: [PATCH 3/3] only use sync copy on AMDGPU; improve logging

---
 offload/libompaccsupport/device.cpp | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 5698340083c756..fae218efe23a59 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -772,17 +772,20 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion,
     freeAfterSynchronization(Device, AsyncInfo, *ReductionAllocOrErr);
   }
 
-  INFO(OMP_INFOTYPE_DATA_TRANSFER, Device.DeviceID,
-       "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
-       ", Size=%" PRId64 ", Name=KernelLaunchEnv\n",
-       DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
-       sizeof(KernelLaunchEnvironmentTy));
-
   // The allocation is fresh, so no work pending on the queue accesses it and
-  // the copy can bypass the queue. Without a queue, the plugin may write the
-  // memory directly.
-  if (Device.submitDataSync(*AllocOrErr, &LocalKLE,
-                            sizeof(KernelLaunchEnvironmentTy)))
+  // the copy can bypass the queue. Without a queue, the AMDGPU plugin may write
+  // the memory directly. Other backends copy on the queue.
+  int32_t Ret =
+      Device.getBackend() == OL_PLATFORM_BACKEND_AMDGPU
+          ? Device.submitDataSync(*AllocOrErr, &LocalKLE,
+                                  sizeof(KernelLaunchEnvironmentTy),
+                                  /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr,
+                                  /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv")
+          : Device.submitData(*AllocOrErr, &LocalKLE,
+                              sizeof(KernelLaunchEnvironmentTy), AsyncInfo,
+                              /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr,
+                              /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv");
+  if (Ret)
     return createError(ErrorCode::BackendFailure,
                        "failed to copy the kernel launch environment to "
                        "device %d",



More information about the llvm-branch-commits mailing list