[clang] [CIR] Make LoweringPrepare safe to rerun on CUDA/HIP host modules (PR #237413)

Konstantinos Parasyris via cfe-commits cfe-commits at lists.llvm.org
Tue Nov 3 12:51:53 PST 2026


https://github.com/koparasy updated https://github.com/llvm/llvm-project/pull/237413

>From 83af50b9a14ed481d4255ca1f37ea4017eac717d Mon Sep 17 00:00:00 2001
From: Konstantinos Parasyris <konstantinos.parasyris at intel.com>
Date: Fri, 30 Oct 2026 14:54:27 -0700
Subject: [PATCH 1/2] [CIR] Make LoweringPrepare safe to rerun on CUDA/HIP host
 modules

LoweringPrepare registers kernels and device variables with the runtime
from the 'cu.kernel_name' attribute on kernel stubs and the
'cu.var_registration' attribute on device variables, but left both in
place. Running the pass again on its own output, e.g. on an emitted
.cir file, rebuilt the module ctor and asserted with "function already
has an entry block" for HIP. CUDA only escaped because the pass already
drops the device binary attribute, without which CUDA builds no ctor;
HIP builds one regardless.

Drop both attributes once the module ctor is built. Only CIRGen and
LoweringPrepare touch them, and after the pass the registration is in
the IR. Only the host side changes.
---
 .../Dialect/Transforms/LoweringPrepare.cpp    |  7 ++++
 clang/test/CIR/CodeGenCUDA/address-spaces.cu  | 17 +++++---
 .../test/CIR/CodeGenCUDA/kernel-stub-name.cu  | 14 ++++---
 clang/test/CIR/CodeGenCUDA/surface.cu         |  8 +++-
 .../CodeGenHIP/device-var-registration.hip    | 11 ++++--
 clang/test/CIR/CodeGenHIP/simple.cpp          |  8 ++--
 ...wering-prepare-cuda-registration-rerun.cir | 39 +++++++++++++++++++
 7 files changed, 84 insertions(+), 20 deletions(-)
 create mode 100644 clang/test/CIR/Transforms/lowering-prepare-cuda-registration-rerun.cir

diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
index f01b463138d084..e5eda8e2edae1c 100644
--- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
@@ -3498,6 +3498,13 @@ void LoweringPreparePass::runOnOperation() {
     // happen out here because the ctor and the CUDA dtor builder test the
     // attribute to decide whether a device-side binary exists at all.
     mlirModule->removeAttr(CIRDialect::getCUDADeviceBinaryAttrName());
+    // The kernels and device variables have been registered from these.
+    // Nothing after this pass reads them, so drop them to keep a second run
+    // of this pass from registering them again.
+    for (auto &entry : cudaKernelMap)
+      entry.getValue()->removeAttr(CUDAKernelNameAttr::getMnemonic());
+    for (auto &[global, regAttr] : cudaDeviceVars)
+      global->removeAttr(CUDAVarRegistrationInfoAttr::getMnemonic());
   }
 
   buildGlobalCtorDtorList();
diff --git a/clang/test/CIR/CodeGenCUDA/address-spaces.cu b/clang/test/CIR/CodeGenCUDA/address-spaces.cu
index d182d76bc0d9de..e75ec19a285f39 100644
--- a/clang/test/CIR/CodeGenCUDA/address-spaces.cu
+++ b/clang/test/CIR/CodeGenCUDA/address-spaces.cu
@@ -30,8 +30,9 @@
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \
 // RUN:            -x cuda -emit-cir -target-sdk-version=12.3 \
-// RUN:            -I%S/Inputs/ %s -o %t.cir
-// RUN: FileCheck --check-prefix=CIR-HOST --input-file=%t.cir %s
+// RUN:            -I%S/Inputs/ -mmlir --mlir-print-ir-before=cir-lowering-prepare %s -o %t.cir 2> %t-before.cir
+// RUN: FileCheck --check-prefix=CIR-HOST-BEFORE-LPP --input-file=%t-before.cir %s
+// RUN: FileCheck --check-prefix=CIR-HOST --input-file=%t.cir %s --implicit-check-not=cu.var_registration
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \
 // RUN:            -x cuda -emit-llvm -target-sdk-version=12.3 \
@@ -54,7 +55,8 @@
 // CIR-POST: cir.global external  target_address_space(1) @i = #cir.int<0>
 // LLVM-DEVICE-DAG: @i = addrspace(1) {{.*}}global i32 0
 // OGCG-DAG: @i = addrspace(1) externally_initialized global i32 0
-// CIR-HOST: cir.global {{.*}} @i = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<i, Variable>}
+// CIR-HOST: cir.global {{.*}} @i = #cir.undef : {{.*}}
+// CIR-HOST-BEFORE-LPP: cir.global {{.*}} @i = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<i, Variable>}
 // LLVM-HOST: @i = internal global i32 undef
 // OGCG-HOST: @i = internal global i32 undef
 __device__ int i;
@@ -64,7 +66,8 @@ __device__ int i;
 // CIR-DEVICE: cir.global constant external target_address_space(4) @j = #cir.int<0> : {{.*}} {{.*}}{cu.externally_initialized = #cir.cu.externally_initialized, cu.var_registration = #cir.cu.var_registration<j, Variable, constant>}
 // LLVM-DEVICE-DAG: @j = addrspace(4) {{.*}}constant i32 0
 // OGCG-DAG: @j = addrspace(4) externally_initialized constant i32 0
-// CIR-HOST:  cir.global {{.*}} @j = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<j, Variable, constant>}
+// CIR-HOST:  cir.global {{.*}} @j = #cir.undef : {{.*}}
+// CIR-HOST-BEFORE-LPP:  cir.global {{.*}} @j = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<j, Variable, constant>}
 // LLVM-HOST: @j = internal global i32 undef
 // OGCG-HOST: @j = internal global i32 undef
 __constant__ int j;
@@ -102,7 +105,8 @@ extern __constant__ int ext_constant_var;
 // External device variables with definitions should be internal on host
 // CIR-DEVICE: cir.global external target_address_space(1) @ext_device_var_def = #cir.int<1>
 // LLVM-DEVICE: @ext_device_var_def = addrspace(1) externally_initialized global i32 1, align 4
-// CIR-HOST: cir.global "private" internal {{.*}} @ext_device_var_def = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<ext_device_var_def, Variable>}
+// CIR-HOST: cir.global "private" internal {{.*}} @ext_device_var_def = #cir.undef : {{.*}}
+// CIR-HOST-BEFORE-LPP: cir.global "private" internal {{.*}} @ext_device_var_def = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<ext_device_var_def, Variable>}
 // LLVM-HOST: @ext_device_var_def = internal global i32 undef, align 4
 // OGCG-HOST: @ext_device_var_def = internal global i32
 // OGCG-DEVICE: @ext_device_var_def = addrspace(1) externally_initialized global i32 1, align 4
@@ -113,7 +117,8 @@ __device__ int ext_device_var_def = 1;
 // LLVM-DEVICE: @ext_constant_var_def = addrspace(4) externally_initialized constant i32 2, align 4
 // OGCG-DEVICE: @ext_constant_var_def = addrspace(4) externally_initialized constant i32 2, align 4
 // OGCG-DEVICE: @_ZZ8fnStructvE1s = internal addrspace(3) global %struct.S undef
-// CIR-HOST: cir.global "private" internal {{.*}} @ext_constant_var_def = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<ext_constant_var_def, Variable, constant>}
+// CIR-HOST: cir.global "private" internal {{.*}} @ext_constant_var_def = #cir.undef : {{.*}}
+// CIR-HOST-BEFORE-LPP: cir.global "private" internal {{.*}} @ext_constant_var_def = #cir.undef : {{.*}} {{.*}}{cu.var_registration = #cir.cu.var_registration<ext_constant_var_def, Variable, constant>}
 // LLVM-HOST: @ext_constant_var_def = internal global i32 undef, align 4
 // OGCG-HOST: @ext_constant_var_def = internal global i32
 extern __constant__ int ext_constant_var_def;
diff --git a/clang/test/CIR/CodeGenCUDA/kernel-stub-name.cu b/clang/test/CIR/CodeGenCUDA/kernel-stub-name.cu
index 045b836bdbe47e..865f3f779c620b 100644
--- a/clang/test/CIR/CodeGenCUDA/kernel-stub-name.cu
+++ b/clang/test/CIR/CodeGenCUDA/kernel-stub-name.cu
@@ -1,22 +1,26 @@
 // Based on clang/test/CodeGenCUDA/kernel-stub-name.cu.
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-sdk-version=9.2 \
-// RUN:   -emit-cir %s -x cuda -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s
+// RUN:   -emit-cir -mmlir --mlir-print-ir-before=cir-lowering-prepare %s -x cuda -o %t.cir 2> %t-before.cir
+// RUN: FileCheck --input-file=%t-before.cir %s --check-prefix=CIR-BEFORE-LPP
+// RUN: FileCheck --input-file=%t.cir %s --implicit-check-not=cu.kernel_name
 
 #include "Inputs/cuda.h"
 
-// CHECK: cir.func {{.*}} @[[CSTUB:__device_stub__ckernel]]() attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"ckernel">{{.*}}}
+// CHECK: cir.func {{.*}} @[[CSTUB:__device_stub__ckernel]]()
+// CIR-BEFORE-LPP: cir.func {{.*}} @[[CSTUB:__device_stub__ckernel]]() attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"ckernel">{{.*}}}
 // CHECK: cir.return
 // CHECK-NEXT: }
 extern "C" __global__ void ckernel() {}
 
-// CHECK: cir.func {{.*}} @_ZN2ns23__device_stub__nskernelEv() attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"_ZN2ns8nskernelEv">{{.*}}}
+// CHECK: cir.func {{.*}} @_ZN2ns23__device_stub__nskernelEv()
+// CIR-BEFORE-LPP: cir.func {{.*}} @_ZN2ns23__device_stub__nskernelEv() attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"_ZN2ns8nskernelEv">{{.*}}}
 namespace ns {
 __global__ void nskernel() {}
 } // namespace ns
 
-// CHECK: cir.func {{.*}} @_Z25__device_stub__kernelfuncIiEvv() attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"_Z10kernelfuncIiEvv">{{.*}}}
+// CHECK: cir.func {{.*}} @_Z25__device_stub__kernelfuncIiEvv()
+// CIR-BEFORE-LPP: cir.func {{.*}} @_Z25__device_stub__kernelfuncIiEvv() attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"_Z10kernelfuncIiEvv">{{.*}}}
 template <class T>
 __global__ void kernelfunc() {}
 template __global__ void kernelfunc<int>();
diff --git a/clang/test/CIR/CodeGenCUDA/surface.cu b/clang/test/CIR/CodeGenCUDA/surface.cu
index b4bb5ea0f623ee..7ebbf061a5ea85 100644
--- a/clang/test/CIR/CodeGenCUDA/surface.cu
+++ b/clang/test/CIR/CodeGenCUDA/surface.cu
@@ -8,7 +8,10 @@
 // RUN: echo -n "GPU binary would be here." > %t
 // RUN: %clang_cc1 -fclangir -std=c++11 -triple x86_64-unknown-linux-gnu \
 // RUN:   -target-sdk-version=12.3 -fcuda-include-gpubinary %t \
-// RUN:   -emit-cir -o - %s | FileCheck --check-prefix=CIR-HOST %s
+// RUN:   -emit-cir -o - %s | FileCheck --check-prefix=CIR-HOST %s --implicit-check-not=cu.var_registration
+// RUN: %clang_cc1 -fclangir -std=c++11 -triple x86_64-unknown-linux-gnu \
+// RUN:   -target-sdk-version=12.3 -fcuda-include-gpubinary %t \
+// RUN:   -emit-cir -mmlir --mlir-print-ir-before=cir-lowering-prepare -o /dev/null %s 2>&1 | FileCheck --check-prefix=CIR-HOST-BEFORE-LPP %s
 // RUN: %clang_cc1 -fclangir -std=c++11 -triple x86_64-unknown-linux-gnu \
 // RUN:   -target-sdk-version=12.3 -fcuda-include-gpubinary %t \
 // RUN:   -emit-llvm -o - %s | FileCheck --check-prefix=LLVM-HOST %s
@@ -61,7 +64,8 @@ surface<void, 2> surf;
 
 // Check that the host-side shadow is internalized and carries the surface
 // registration metadata, including the surface type from surface<void, 2>.
-// CIR-HOST: cir.global "private" internal dso_local @surf = {{.*}}cu.var_registration = #cir.cu.var_registration<surf, Surface, surface_type = 2>
+// CIR-HOST: cir.global "private" internal dso_local @surf =
+// CIR-HOST-BEFORE-LPP: cir.global "private" internal dso_local @surf = {{.*}}cu.var_registration = #cir.cu.var_registration<surf, Surface, surface_type = 2>
 // LLVM-HOST: @surf = internal global %{{.*}} zeroinitializer
 
 // Check both CIR-lowered LLVM and original CodeGen registration.
diff --git a/clang/test/CIR/CodeGenHIP/device-var-registration.hip b/clang/test/CIR/CodeGenHIP/device-var-registration.hip
index 77bde0bb42eea7..f83be86105affa 100644
--- a/clang/test/CIR/CodeGenHIP/device-var-registration.hip
+++ b/clang/test/CIR/CodeGenHIP/device-var-registration.hip
@@ -5,8 +5,9 @@
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \
 // RUN:            -x hip -fhip-new-launch-api -fcuda-include-gpubinary %t \
-// RUN:            -emit-cir %s -o %t.cir
-// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN:            -emit-cir -mmlir --mlir-print-ir-before=cir-lowering-prepare %s -o %t.cir 2> %t-before.cir
+// RUN: FileCheck --check-prefix=CIR-BEFORE-LPP --input-file=%t-before.cir %s
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s --implicit-check-not=cu.var_registration
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \
 // RUN:            -x hip -fhip-new-launch-api -fcuda-include-gpubinary %t \
@@ -25,8 +26,10 @@ __attribute__((constant)) int constant_var;
 // CIR-LABEL: cir.func internal private @__hip_register_globals
 // CIR: cir.call @__hipRegisterVar
 // CIR: cir.call @__hipRegisterVar
-// CIR-DAG: cir.global{{.*}} @device_var = {{.*}}cu.var_registration = #cir.cu.var_registration<device_var, Variable>
-// CIR-DAG: cir.global{{.*}} @constant_var = {{.*}}cu.var_registration = #cir.cu.var_registration<constant_var, Variable, constant>
+// CIR-DAG: cir.global{{.*}} @device_var =
+// CIR-BEFORE-LPP-DAG: cir.global{{.*}} @device_var = {{.*}}cu.var_registration = #cir.cu.var_registration<device_var, Variable>
+// CIR-DAG: cir.global{{.*}} @constant_var =
+// CIR-BEFORE-LPP-DAG: cir.global{{.*}} @constant_var = {{.*}}cu.var_registration = #cir.cu.var_registration<constant_var, Variable, constant>
 
 // LLVM-LABEL: define internal void @__hip_register_globals
 // LLVM: call void @__hipRegisterVar(ptr %[[HANDLE:[^,]+]], ptr @device_var, ptr @[[DEVICE_VAR_NAME:[^,]+]], ptr @[[DEVICE_VAR_NAME]], i32 0, i64 4, i32 0, i32 0)
diff --git a/clang/test/CIR/CodeGenHIP/simple.cpp b/clang/test/CIR/CodeGenHIP/simple.cpp
index 5902f95b9167f6..ec7fe51c8d8de7 100644
--- a/clang/test/CIR/CodeGenHIP/simple.cpp
+++ b/clang/test/CIR/CodeGenHIP/simple.cpp
@@ -2,8 +2,9 @@
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir \
 // RUN:            -x hip -fhip-new-launch-api \
-// RUN:            -I%S/../CodeGenCUDA/Inputs/ -emit-cir %s -o %t.cir
-// RUN: FileCheck --check-prefix=CIR-HOST --input-file=%t.cir %s
+// RUN:            -I%S/../CodeGenCUDA/Inputs/ -emit-cir -mmlir --mlir-print-ir-before=cir-lowering-prepare %s -o %t.cir 2> %t-before.cir
+// RUN: FileCheck --check-prefix=CIR-HOST-BEFORE-LPP --input-file=%t-before.cir %s
+// RUN: FileCheck --check-prefix=CIR-HOST --input-file=%t.cir %s --implicit-check-not=cu.kernel_name
 
 // RUN: %clang_cc1 -triple=amdgpu-amd-amdhsa -x hip -fclangir \
 // RUN:            -fcuda-is-device -fhip-new-launch-api \
@@ -42,7 +43,8 @@ __global__ void global_fn(int a) {}
 // CIR-DEVICE: cir.func {{.*}}{{.*}} @_Z9global_fni
 // OGCG-DEVICE: define protected amdgpu_kernel void @_Z9global_fni
 
-// CIR-HOST: @_Z24__device_stub__global_fni{{.*}}attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"_Z9global_fni">{{.*}}}
+// CIR-HOST: @_Z24__device_stub__global_fni
+// CIR-HOST-BEFORE-LPP: @_Z24__device_stub__global_fni{{.*}}attributes {{{.*}}cu.kernel_name = #cir.cu.kernel_name<"_Z9global_fni">{{.*}}}
 // CIR-HOST: %[[#CIRKernelArgs:]] = cir.alloca {{.*}}"kernel_args"
 // CIR-HOST: %[[#Decayed:]] = cir.cast array_to_ptrdecay %[[#CIRKernelArgs]]
 // CIR-HOST: cir.call @__hipPopCallConfiguration
diff --git a/clang/test/CIR/Transforms/lowering-prepare-cuda-registration-rerun.cir b/clang/test/CIR/Transforms/lowering-prepare-cuda-registration-rerun.cir
new file mode 100644
index 00000000000000..5ec274ea9a7ffd
--- /dev/null
+++ b/clang/test/CIR/Transforms/lowering-prepare-cuda-registration-rerun.cir
@@ -0,0 +1,39 @@
+// RUN: cir-opt %s --cir-lowering-prepare -o %t.hip.once.cir
+// RUN: cir-opt %t.hip.once.cir --cir-lowering-prepare -o %t.hip.twice.cir
+// RUN: diff %t.hip.once.cir %t.hip.twice.cir
+// RUN: FileCheck %s --input-file=%t.hip.once.cir --check-prefixes=CHECK,HIP \
+// RUN:   --implicit-check-not=cu.kernel_name --implicit-check-not=cu.var_registration
+
+// RUN: sed -e 's/hip = true/hip = false/' \
+// RUN:     -e 's/module attributes {/module attributes {cir.cu.device_binary = "abc" : !cir.array<!u8i x 3>, /' \
+// RUN:     %s | cir-opt --cir-lowering-prepare -o %t.cuda.once.cir
+// RUN: cir-opt %t.cuda.once.cir --cir-lowering-prepare -o %t.cuda.twice.cir
+// RUN: diff %t.cuda.once.cir %t.cuda.twice.cir
+// RUN: FileCheck %s --input-file=%t.cuda.once.cir --check-prefixes=CHECK,CUDA \
+// RUN:   --implicit-check-not=cu.kernel_name --implicit-check-not=cu.var_registration
+
+// LoweringPrepare consumes 'cu.kernel_name' and 'cu.var_registration' when it
+// registers the kernels and device variables with the runtime, so running it
+// again on its own output registers nothing a second time. HIP builds the
+// module ctor without a device-side binary; CUDA needs one.
+
+!s32i = !cir.int<s, 32>
+!u8i = !cir.int<u, 8>
+module attributes {cir.lowering_lang_options = #cir.lowering_lang_options<exceptions = false, threadsafe_statics = true, cuda = true, cuda_is_device = false, hip = true, hlsl = false, gpu_rdc = false, openmp = false, openmp_is_target_device = false, clang_abi_compat = 16, long_double_size = 0>, cir.triple = "x86_64-unknown-linux-gnu"} {
+  cir.global "private" internal dso_local @dv = #cir.undef : !s32i align(4) {cu.var_registration = #cir.cu.var_registration<dv, Variable>}
+  cir.global constant external @_Z1kv = #cir.global_view<@_Z16__device_stub__kv> : !cir.ptr<!cir.func<()>> align(8)
+  cir.func @_Z16__device_stub__kv() attributes {cu.kernel_name = #cir.cu.kernel_name<"_Z1kv">} {
+    cir.return
+  }
+}
+
+// HIP:  cir.func internal private @__hip_register_globals
+// HIP:    cir.call @__hipRegisterFunction
+// HIP:    cir.call @__hipRegisterVar
+// HIP:  cir.func internal private @__hip_module_ctor()
+// CUDA: cir.func internal private @__cuda_register_globals
+// CUDA:   cir.call @__cudaRegisterFunction
+// CUDA:   cir.call @__cudaRegisterVar
+// CUDA: cir.func internal private @__cuda_module_ctor()
+// CHECK: cir.global "private" internal dso_local @dv = #cir.undef : !s32i align(4)
+// CHECK: cir.func @_Z16__device_stub__kv() {

>From 364ddb45cf41a4aa95c9669377e5b80b3cbfe59c Mon Sep 17 00:00:00 2001
From: Konstantinos Parasyris <konstantinos.parasyris at intel.com>
Date: Tue, 3 Nov 2026 11:18:13 -0800
Subject: [PATCH 2/2] Fix comment

---
 clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
index e5eda8e2edae1c..d15ccea7feadee 100644
--- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
@@ -3503,8 +3503,8 @@ void LoweringPreparePass::runOnOperation() {
     // of this pass from registering them again.
     for (auto &entry : cudaKernelMap)
       entry.getValue()->removeAttr(CUDAKernelNameAttr::getMnemonic());
-    for (auto &[global, regAttr] : cudaDeviceVars)
-      global->removeAttr(CUDAVarRegistrationInfoAttr::getMnemonic());
+    for (auto &deviceVar : cudaDeviceVars)
+      deviceVar.first->removeAttr(CUDAVarRegistrationInfoAttr::getMnemonic());
   }
 
   buildGlobalCtorDtorList();



More information about the cfe-commits mailing list