[Mlir-commits] [mlir] [MLIR][ExecutionEngine] Tolerate CUDA_ERROR_DEINITIALIZED in mgpuModuleUnload (PR #190563)

[Jared Hoberock]

https://github.com/jaredhoberock updated https://github.com/llvm/llvm-project/pull/190563

>From 85a7ec62cc20a9697596a351a0d9d177dcff3b5a Mon Sep 17 00:00:00 2001
From: Jared Hoberock <jaredhoberock at gmail.com>
Date: Mon, 6 Apr 2026 12:53:50 -0500
Subject: [PATCH] [MLIR][ExecutionEngine] Tolerate CUDA context shutdown errors
 in resource cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

During program exit, CUDA resource cleanup functions (module unload,
stream/event destroy, etc.) may be called after the CUDA primary context
has already been destroyed. This produces spurious error messages like
'cuModuleUnload(module)' failed with 'CUDA_ERROR_DEINITIALIZED' or
'cuEventDestroy(event)' failed with 'CUDA_ERROR_CONTEXT_IS_DESTROYED'.

These errors are benign — all resources are already freed when the
context is destroyed.

Add CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN macro that silences both
CUDA_ERROR_DEINITIALIZED (4) and CUDA_ERROR_CONTEXT_IS_DESTROYED (709),
and apply it to the five cleanup wrappers: mgpuModuleUnload,
mgpuStreamDestroy, mgpuStreamWaitEvent, mgpuEventDestroy, and
mgpuEventSynchronize.

Re-enable FileCheck in async.mlir integration test.

Fixes #170833, fixes #190563.
---
 .../ExecutionEngine/CudaRuntimeWrappers.cpp   | 33 ++++++++++++++++---
 mlir/test/Integration/GPU/CUDA/async.mlir     |  7 ++--
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
index 6307e0b59f3d2..4363e841d69a7 100644
--- a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
+++ b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
@@ -45,6 +45,29 @@
     fprintf(stderr, "'%s' failed with '%s'\n", #expr, name);                   \
   }(expr)
 
+/// Helper to check if a CUDA error is due to the context being destroyed
+/// during program shutdown. Both CUDA_ERROR_DEINITIALIZED and
+/// CUDA_ERROR_CONTEXT_IS_DESTROYED indicate that the CUDA context has been
+/// torn down and any associated resources are already freed.
+static bool isCudaContextShutdownError(CUresult result) {
+  return result == CUDA_ERROR_DEINITIALIZED ||
+         result == CUDA_ERROR_CONTEXT_IS_DESTROYED;
+}
+
+/// Like CUDA_REPORT_IF_ERROR, but silences errors caused by CUDA context
+/// shutdown. These errors are benign when they occur during program exit,
+/// as all resources are freed with the context.
+#define CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(expr)                             \
+  [](CUresult result) {                                                        \
+    if (!result || isCudaContextShutdownError(result))                         \
+      return;                                                                  \
+    const char *name = nullptr;                                                \
+    cuGetErrorName(result, &name);                                             \
+    if (!name)                                                                 \
+      name = "<unknown>";                                                      \
+    fprintf(stderr, "'%s' failed with '%s'\n", #expr, name);                   \
+  }(expr)
+
 #define CUSPARSE_REPORT_IF_ERROR(expr)                                         \
   {                                                                            \
     cusparseStatus_t status = (expr);                                          \
@@ -146,7 +169,7 @@ mgpuModuleLoadJIT(void *data, int optLevel, size_t /*assmeblySize*/) {
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuModuleUnload(CUmodule module) {
-  CUDA_REPORT_IF_ERROR(cuModuleUnload(module));
+  CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuModuleUnload(module));
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUfunction
@@ -199,7 +222,7 @@ extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUstream mgpuStreamCreate() {
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuStreamDestroy(CUstream stream) {
-  CUDA_REPORT_IF_ERROR(cuStreamDestroy(stream));
+  CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuStreamDestroy(stream));
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
@@ -209,7 +232,7 @@ mgpuStreamSynchronize(CUstream stream) {
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuStreamWaitEvent(CUstream stream,
                                                               CUevent event) {
-  CUDA_REPORT_IF_ERROR(cuStreamWaitEvent(stream, event, /*flags=*/0));
+  CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuStreamWaitEvent(stream, event, /*flags=*/0));
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUevent mgpuEventCreate() {
@@ -220,11 +243,11 @@ extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUevent mgpuEventCreate() {
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventDestroy(CUevent event) {
-  CUDA_REPORT_IF_ERROR(cuEventDestroy(event));
+  CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuEventDestroy(event));
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventSynchronize(CUevent event) {
-  CUDA_REPORT_IF_ERROR(cuEventSynchronize(event));
+  CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuEventSynchronize(event));
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventRecord(CUevent event,
diff --git a/mlir/test/Integration/GPU/CUDA/async.mlir b/mlir/test/Integration/GPU/CUDA/async.mlir
index 3e45b5af5826c..5acadd61f432c 100644
--- a/mlir/test/Integration/GPU/CUDA/async.mlir
+++ b/mlir/test/Integration/GPU/CUDA/async.mlir
@@ -8,11 +8,8 @@
 // RUN:   --shared-libs=%mlir_cuda_runtime \
 // RUN:   --shared-libs=%mlir_async_runtime \
 // RUN:   --shared-libs=%mlir_runner_utils \
-// RUN:   --entry-point-result=void -O0
-// RUN:
-// This test is overly flaky right now and needs investigation, skipping FileCheck.
-// See: https://github.com/llvm/llvm-project/issues/170833
-// DISABLED: | FileCheck %s
+// RUN:   --entry-point-result=void -O0 \
+// RUN: | FileCheck %s
 
 func.func @main() {
   %c0    = arith.constant 0 : index