[Mlir-commits] [mlir] 7087ece - [MLIR][ExecutionEngine] Tolerate CUDA_ERROR_DEINITIALIZED in mgpuModuleUnload (#190563)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Mon Apr 6 14:12:04 PDT 2026
Author: Jared Hoberock
Date: 2026-04-06T21:11:58Z
New Revision: 7087ece044f1313206ce5d5e5115862fe29143c1
URL: https://github.com/llvm/llvm-project/commit/7087ece044f1313206ce5d5e5115862fe29143c1
DIFF: https://github.com/llvm/llvm-project/commit/7087ece044f1313206ce5d5e5115862fe29143c1.diff
LOG: [MLIR][ExecutionEngine] Tolerate CUDA_ERROR_DEINITIALIZED in mgpuModuleUnload (#190563)
`mgpuModuleUnload` may be called from a global destructor (registered by
`SelectObjectAttr`'s `appendToGlobalDtors`) after the CUDA primary
context has already been destroyed during program shutdown. In this
case, `cuModuleUnload` returns `CUDA_ERROR_DEINITIALIZED`, which is
benign since the module's resources are already freed with the context.
## Reproduction
Any program that uses `gpu.launch_func` and is AOT-compiled (via
`mlir-translate --mlir-to-llvmir | llc | cc -lmlir_cuda_runtime`) will
print `'cuModuleUnload(module)' failed with '<unknown>'` on exit. This
is because `SelectObjectAttr` registers the module unload as a global
destructor, which runs after the CUDA primary context is released.
This script reproduces the error message from `mgpuModuleUnload` on my
system:
```
#!/bin/bash
set -e
LLVM_BUILD=${LLVM_BUILD:-$HOME/dev/git/llvm-project-22/build}
cat > /tmp/repro.mlir << 'MLIR'
func.func @main() {
%c1 = arith.constant 1 : index
gpu.launch blocks(%bx, %by, %bz) in (%gx = %c1, %gy = %c1, %gz = %c1)
threads(%tx, %ty, %tz) in (%bsx = %c1, %bsy = %c1, %bsz = %c1) {
gpu.terminator
}
return
}
MLIR
$LLVM_BUILD/bin/mlir-opt /tmp/repro.mlir \
-gpu-lower-to-nvvm-pipeline="cubin-format=fatbin" \
| $LLVM_BUILD/bin/mlir-translate --mlir-to-llvmir -o /tmp/repro.ll
$LLVM_BUILD/bin/llc -relocation-model=pic -filetype=obj /tmp/repro.ll -o /tmp/repro.o
cc /tmp/repro.o \
-L$LLVM_BUILD/lib -Wl,-rpath,$LLVM_BUILD/lib \
-lmlir_cuda_runtime -lmlir_runner_utils -o /tmp/repro
echo "Running:"
/tmp/repro 2>&1
echo "Exit code: $?"
```
## Context
This matches how other projects handle the same shutdown ordering issue:
- Clang CUDA (D48613) switched module cleanup from
`__attribute__((destructor))` to `atexit()`
- GCC libgomp checks context validity before `cuModuleUnload`
- Apache TVM silently ignores `CUDA_ERROR_DEINITIALIZED` on module
unload
Fixes #170833
Added:
Modified:
mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
mlir/test/Integration/GPU/CUDA/async.mlir
Removed:
################################################################################
diff --git a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
index 6307e0b59f3d2..f60db7b760aa6 100644
--- a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
+++ b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
@@ -45,6 +45,29 @@
fprintf(stderr, "'%s' failed with '%s'\n", #expr, name); \
}(expr)
+/// Helper to check if a CUDA error is due to the context being destroyed
+/// during program shutdown. Both CUDA_ERROR_DEINITIALIZED and
+/// CUDA_ERROR_CONTEXT_IS_DESTROYED indicate that the CUDA context has been
+/// torn down and any associated resources are already freed.
+static bool isCudaContextShutdownError(CUresult result) {
+ return result == CUDA_ERROR_DEINITIALIZED ||
+ result == CUDA_ERROR_CONTEXT_IS_DESTROYED;
+}
+
+/// Like CUDA_REPORT_IF_ERROR, but silences errors caused by CUDA context
+/// shutdown. These errors are benign when they occur during program exit,
+/// as all resources are freed with the context.
+#define CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(expr) \
+ [](CUresult result) { \
+ if (!result || isCudaContextShutdownError(result)) \
+ return; \
+ const char *name = nullptr; \
+ cuGetErrorName(result, &name); \
+ if (!name) \
+ name = "<unknown>"; \
+ fprintf(stderr, "'%s' failed with '%s'\n", #expr, name); \
+ }(expr)
+
#define CUSPARSE_REPORT_IF_ERROR(expr) \
{ \
cusparseStatus_t status = (expr); \
@@ -146,7 +169,7 @@ mgpuModuleLoadJIT(void *data, int optLevel, size_t /*assmeblySize*/) {
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuModuleUnload(CUmodule module) {
- CUDA_REPORT_IF_ERROR(cuModuleUnload(module));
+ CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuModuleUnload(module));
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUfunction
@@ -199,7 +222,7 @@ extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUstream mgpuStreamCreate() {
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuStreamDestroy(CUstream stream) {
- CUDA_REPORT_IF_ERROR(cuStreamDestroy(stream));
+ CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuStreamDestroy(stream));
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
@@ -209,7 +232,8 @@ mgpuStreamSynchronize(CUstream stream) {
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuStreamWaitEvent(CUstream stream,
CUevent event) {
- CUDA_REPORT_IF_ERROR(cuStreamWaitEvent(stream, event, /*flags=*/0));
+ CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(
+ cuStreamWaitEvent(stream, event, /*flags=*/0));
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUevent mgpuEventCreate() {
@@ -220,11 +244,11 @@ extern "C" MLIR_CUDA_WRAPPERS_EXPORT CUevent mgpuEventCreate() {
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventDestroy(CUevent event) {
- CUDA_REPORT_IF_ERROR(cuEventDestroy(event));
+ CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuEventDestroy(event));
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventSynchronize(CUevent event) {
- CUDA_REPORT_IF_ERROR(cuEventSynchronize(event));
+ CUDA_REPORT_IF_ERROR_IGNORE_SHUTDOWN(cuEventSynchronize(event));
}
extern "C" MLIR_CUDA_WRAPPERS_EXPORT void mgpuEventRecord(CUevent event,
diff --git a/mlir/test/Integration/GPU/CUDA/async.mlir b/mlir/test/Integration/GPU/CUDA/async.mlir
index 3e45b5af5826c..5acadd61f432c 100644
--- a/mlir/test/Integration/GPU/CUDA/async.mlir
+++ b/mlir/test/Integration/GPU/CUDA/async.mlir
@@ -8,11 +8,8 @@
// RUN: --shared-libs=%mlir_cuda_runtime \
// RUN: --shared-libs=%mlir_async_runtime \
// RUN: --shared-libs=%mlir_runner_utils \
-// RUN: --entry-point-result=void -O0
-// RUN:
-// This test is overly flaky right now and needs investigation, skipping FileCheck.
-// See: https://github.com/llvm/llvm-project/issues/170833
-// DISABLED: | FileCheck %s
+// RUN: --entry-point-result=void -O0 \
+// RUN: | FileCheck %s
func.func @main() {
%c0 = arith.constant 0 : index
More information about the Mlir-commits
mailing list