[clang] [llvm] [Clang][NVPTX] Add __isGridConstant() support (PR #242563)
via cfe-commits
cfe-commits at lists.llvm.org
Fri Nov 13 18:23:06 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-nvptx
Author: losewayy (losewayy)
<details>
<summary>Changes</summary>
This patch adds the missing Clang support for CUDA's `__isGridConstant()` intrinsic.
CUDA's `sm_20_intrinsics.hpp` implements `__isGridConstant()` by calling `__nv_isGridConstant_impl`, which nvcc lowers to `isspacep.param`. Clang has no definition for that symbol, so device code that calls `__isGridConstant()` fails at the PTX assembly stage with:
ptxas fatal : Unresolved extern function '__nv_isGridConstant_impl'
Clang already redefines the four sibling address-space predicates (`__isGlobal`, `__isShared`, `__isConstant`, `__isLocal`) in `__clang_cuda_runtime_wrapper.h`. This patch adds the missing grid-constant sibling:
- `llvm.nvvm.isspacep.grid_const`, lowered to `isspacep.param` and named after the NVVM IR specification
- `__nvvm_isspacep_grid_const`, gated on sm_70 and PTX 7.7
- `__isGridConstant` in `__clang_cuda_runtime_wrapper.h`
- the intrinsic in the NVPTX usage documentation
The intrinsic intentionally does not use `NoCapture`. If it did, a plain byval parameter could be inferred `captures(none)` and `readonly`, and `NVPTXLowerArgs`' byval copy-avoidance optimization could leave it in the parameter address space at -O1/-O2. `isspacep.param` would then return 1 for a parameter that is not annotated `__grid_constant__`. With the current form, a plain byval parameter is copied to local memory before the query, matching nvcc.
Tests:
- `llvm/test/CodeGen/NVPTX/isspacep-grid-const.ll` checks `isspacep.param` selection for sm_90 and for the sm_70/PTX 7.7 minimum.
- `clang/test/CodeGenCUDA/builtins-grid-const.cu` checks that the builtin lowers to the intrinsic and that both the SM and PTX feature gates reject unsupported targets.
- `clang/test/Headers/cuda-grid-constant.cu` runs the CUDA wrapper at -O2, exercises the macro rename, and checks that a grid-constant byval parameter stays in the parameter space while a plain byval parameter is copied to local memory.
The three new tests pass, as do the related NVPTX and Clang CUDA tests. The original failure was reproduced with unpatched Clang 22.1.8 and CUDA 13.4.3 on sm_120; the patched compiler emits `cvta.param` and `isspacep.param`, and ptxas accepts the result.
Runtime validation on sm_120 with CUDA 13.4.3 at both -O0 and -O2 gives:
direct: gc=1 sub=1 plain=0 copy=0 local=0 glob=0 shared=0 scalar=0
escape: gc=1 plain=0 local=0
The CUDA `char4` type from the issue also returns 1 for `&v` and `&v.x`, matching nvcc 13.4.
Known limitation: scalar and native Clang vector `__grid_constant__` parameters still use a local copy and return 0. Scalar addressing is tracked in #<!-- -->230158 (PR #<!-- -->231567 or an equivalent backend fix); this patch covers byval parameters, including CUDA vector types such as `char4`.
Part of #<!-- -->230158.
Assisted-by: Claude Code
---
Full diff: https://github.com/llvm/llvm-project/pull/242563.diff
9 Files Affected:
- (modified) clang/include/clang/Basic/BuiltinsNVPTX.td (+1)
- (modified) clang/lib/Headers/__clang_cuda_runtime_wrapper.h (+8)
- (added) clang/test/CodeGenCUDA/builtins-grid-const.cu (+17)
- (added) clang/test/Headers/Inputs/include-grid-constant/sm_20_intrinsics.hpp (+8)
- (added) clang/test/Headers/cuda-grid-constant.cu (+31)
- (modified) llvm/docs/NVPTXUsage.md (+6)
- (modified) llvm/include/llvm/IR/IntrinsicsNVVM.td (+8)
- (modified) llvm/lib/Target/NVPTX/NVPTXIntrinsics.td (+2)
- (added) llvm/test/CodeGen/NVPTX/isspacep-grid-const.ll (+12)
``````````diff
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.td b/clang/include/clang/Basic/BuiltinsNVPTX.td
index 904cd3c555c30..aab12f0f3a811 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.td
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.td
@@ -1415,6 +1415,7 @@ let Attributes = [NoThrow, Const] in {
def __nvvm_isspacep_global : NVPTXBuiltin<"bool(void const *)">;
def __nvvm_isspacep_local : NVPTXBuiltin<"bool(void const *)">;
def __nvvm_isspacep_shared : NVPTXBuiltin<"bool(void const *)">;
+ def __nvvm_isspacep_grid_const : NVPTXBuiltinSMAndPTX<"bool(void const *)", SM_70, PTX77>;
def __nvvm_isspacep_shared_cluster : NVPTXBuiltinSMAndPTX<"bool(void const *)", SM_90, PTX78>;
}
diff --git a/clang/lib/Headers/__clang_cuda_runtime_wrapper.h b/clang/lib/Headers/__clang_cuda_runtime_wrapper.h
index 29178bf0db8a1..c050df25c1ead 100644
--- a/clang/lib/Headers/__clang_cuda_runtime_wrapper.h
+++ b/clang/lib/Headers/__clang_cuda_runtime_wrapper.h
@@ -292,15 +292,18 @@ static inline __device__ void __brkpt(int __c) { __brkpt(); }
#pragma push_macro("__isShared")
#pragma push_macro("__isConstant")
#pragma push_macro("__isLocal")
+#pragma push_macro("__isGridConstant")
#define __isGlobal __ignored_cuda___isGlobal
#define __isShared __ignored_cuda___isShared
#define __isConstant __ignored_cuda___isConstant
#define __isLocal __ignored_cuda___isLocal
+#define __isGridConstant __ignored_cuda___isGridConstant
#include "sm_20_intrinsics.hpp"
#pragma pop_macro("__isGlobal")
#pragma pop_macro("__isShared")
#pragma pop_macro("__isConstant")
#pragma pop_macro("__isLocal")
+#pragma pop_macro("__isGridConstant")
#pragma push_macro("__DEVICE__")
#define __DEVICE__ static __device__ __forceinline__ __attribute__((const))
__DEVICE__ unsigned int __isGlobal(const void *p) {
@@ -315,6 +318,11 @@ __DEVICE__ unsigned int __isConstant(const void *p) {
__DEVICE__ unsigned int __isLocal(const void *p) {
return __nvvm_isspacep_local(p);
}
+#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 700)
+__DEVICE__ unsigned int __isGridConstant(const void *p) {
+ return __nvvm_isspacep_grid_const(p);
+}
+#endif
#pragma pop_macro("__DEVICE__")
#include "sm_32_atomic_functions.hpp"
diff --git a/clang/test/CodeGenCUDA/builtins-grid-const.cu b/clang/test/CodeGenCUDA/builtins-grid-const.cu
new file mode 100644
index 0000000000000..487d86a601e72
--- /dev/null
+++ b/clang/test/CodeGenCUDA/builtins-grid-const.cu
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN: -target-feature +ptx77 -emit-llvm -fcuda-is-device -o - %s \
+// RUN: | FileCheck %s
+// RUN: not %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_60 \
+// RUN: -target-feature +ptx77 -emit-llvm -fcuda-is-device -o - %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix=ERROR
+// RUN: not %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN: -target-feature +ptx76 -emit-llvm -fcuda-is-device -o - %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix=ERROR
+
+// CHECK-LABEL: define{{.*}} @_Z4testPKv(
+// CHECK: call i1 @llvm.nvvm.isspacep.grid_const(ptr
+__attribute__((device)) unsigned int test(const void *p) {
+ return __nvvm_isspacep_grid_const(p);
+}
+
+// ERROR: '__nvvm_isspacep_grid_const' needs target feature
diff --git a/clang/test/Headers/Inputs/include-grid-constant/sm_20_intrinsics.hpp b/clang/test/Headers/Inputs/include-grid-constant/sm_20_intrinsics.hpp
new file mode 100644
index 0000000000000..4b924f4b73c25
--- /dev/null
+++ b/clang/test/Headers/Inputs/include-grid-constant/sm_20_intrinsics.hpp
@@ -0,0 +1,8 @@
+// Minimal stand-in for the CUDA header's __isGridConstant definition. The
+// wrapper renames this function before including the header; without the
+// rename, the wrapper's own definition would be a redefinition.
+#pragma once
+
+static __device__ __forceinline__ unsigned int __isGridConstant(const void *p) {
+ return 0;
+}
diff --git a/clang/test/Headers/cuda-grid-constant.cu b/clang/test/Headers/cuda-grid-constant.cu
new file mode 100644
index 0000000000000..7ad42d4b3fa5e
--- /dev/null
+++ b/clang/test/Headers/cuda-grid-constant.cu
@@ -0,0 +1,31 @@
+// REQUIRES: nvptx-registered-target
+// RUN: %clang_cc1 -include __clang_cuda_runtime_wrapper.h \
+// RUN: -internal-isystem %S/../../lib/Headers/cuda_wrappers \
+// RUN: -internal-isystem %S/Inputs/include-grid-constant \
+// RUN: -internal-isystem %S/Inputs/include \
+// RUN: -triple nvptx64-nvidia-cuda -aux-triple x86_64-unknown-unknown \
+// RUN: -target-cpu sm_70 -target-feature +ptx77 -O2 -S \
+// RUN: -fcuda-is-device -o - %s \
+// RUN: -D __CLANG_GPU_DISABLE_MATH_WRAPPERS \
+// RUN: | FileCheck %s \
+// RUN: --implicit-check-not="mov.b64 {{%rd[0-9]+}}, kernel_param_1"
+
+struct S {
+ int x, y;
+};
+
+extern "C" __attribute__((global)) void kernel(
+ __attribute__((grid_constant)) const S gc, const S plain, int *out) {
+ out[0] = __isGridConstant(&gc);
+ out[1] = __isGridConstant(&plain);
+}
+
+// CHECK-LABEL: kernel(
+// The grid-constant byval parameter stays in the parameter address space.
+// CHECK-DAG: mov.b64 %rd{{[0-9]+}}, kernel_param_0
+// CHECK-DAG: cvta.param
+// The plain byval parameter must be copied to local memory. If it is left in
+// the parameter address space, isspacep.param would incorrectly return 1.
+// CHECK-DAG: ld.param.b{{32|64}} {{.*}}, [kernel_param_1
+// CHECK-DAG: st.local.b{{32|64}}
+// CHECK-COUNT-2: isspacep.param
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index acd77a46b4638..de0092421fa8c 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1059,6 +1059,7 @@ declare i1 @llvm.nvvm.isspacep.const(ptr %p)
declare i1 @llvm.nvvm.isspacep.global(ptr %p)
declare i1 @llvm.nvvm.isspacep.local(ptr %p)
declare i1 @llvm.nvvm.isspacep.shared(ptr %p)
+declare i1 @llvm.nvvm.isspacep.grid_const(ptr %p)
declare i1 @llvm.nvvm.isspacep.shared.cluster(ptr %p)
```
@@ -1073,6 +1074,11 @@ If the given pointer in the generic address space refers to memory which falls
within the state space of the intrinsic (and therefore could be safely address
space casted to this space), 1 is returned, otherwise 0 is returned.
+The `llvm.nvvm.isspacep.grid_const` intrinsic queries whether the pointer was
+derived from the address of a kernel parameter that has the
+`"nvvm.grid_constant"` property. It corresponds to the `isspacep.param` PTX
+instruction.
+
#### '`llvm.nvvm.mapa.*`' Intrinsics
##### Syntax:
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 1910f4729ea5f..019e6182a3251 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -2616,6 +2616,14 @@ foreach space = ["const", "global", "local", "shared", "shared_cluster"] in
def int_nvvm_isspacep_ # space : NVVMBuiltin,
NVVMPureIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [NoCapture<ArgIndex<0>>]>;
+// This intrinsic intentionally has no NoCapture. With NoCapture, a plain byval
+// parameter can be inferred captures(none) and readonly, so NVPTXLowerArgs may
+// keep it in the param address space and this query would return true for a
+// parameter that is not grid_constant.
+def int_nvvm_isspacep_grid_const : NVVMBuiltin,
+ NVVMPureIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [],
+ "llvm.nvvm.isspacep.grid_const">;
+
//
// Texture Fetch
//
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 4b523218ddf6b..f686a821318fa 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -3879,6 +3879,8 @@ defm isspace_shared : ISSPACEP<"shared", int_nvvm_isspacep_shared>;
defm isspace_shared_cluster : ISSPACEP<"shared::cluster",
int_nvvm_isspacep_shared_cluster,
[SM90]>;
+defm isspace_grid_const : ISSPACEP<"param", int_nvvm_isspacep_grid_const,
+ [PTX77, SM70]>;
// Special register reads
let isAsCheapAsAMove = true in
diff --git a/llvm/test/CodeGen/NVPTX/isspacep-grid-const.ll b/llvm/test/CodeGen/NVPTX/isspacep-grid-const.ll
new file mode 100644
index 0000000000000..d09375f92944c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/isspacep-grid-const.ll
@@ -0,0 +1,12 @@
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 | %ptxas-verify -arch=sm_90 %}
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx77 | FileCheck %s
+
+declare i1 @llvm.nvvm.isspacep.grid_const(ptr) readnone noinline
+
+; CHECK-LABEL: is_grid_const
+define i1 @is_grid_const(ptr %addr) {
+; CHECK: isspacep.param
+ %v = tail call i1 @llvm.nvvm.isspacep.grid_const(ptr %addr)
+ ret i1 %v
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/242563
More information about the cfe-commits
mailing list