[llvm] [NVVM][NVPTX] Add tensor map override support in S2G and Reduce intrinsics (PR #215503)
Rajat Bajpai via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 02:57:31 PDT 2026
https://github.com/rajatbajpai updated https://github.com/llvm/llvm-project/pull/215503
>From e7604821ce0176d47e56a80fa7a085ba654ab7bf Mon Sep 17 00:00:00 2001
From: rbajpai <rbajpai at nvidia.com>
Date: Tue, 11 Aug 2026 12:45:56 +0530
Subject: [PATCH 1/3] [NVVM][NVPTX] Add tensor map override support in S2G and
Reduce intrinsics
This change adds support for tensor map override in S2G and Reduce
intrinsics. These variants allow overriding tensor map properties.
---
llvm/docs/NVPTXUsage.md | 182 ++++++
llvm/include/llvm/IR/IntrinsicsNVVM.td | 123 +++++
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 174 ++++++
.../cp-async-bulk-tensor-reduce-override.ll | 516 ++++++++++++++++++
.../cp-async-bulk-tensor-s2g-override.ll | 247 +++++++++
5 files changed, 1242 insertions(+)
create mode 100644 llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index 2541d504c6e10..fef89d0a399b7 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -2254,6 +2254,188 @@ functionality as described in the `tile` mode intrinsics above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+#### '`llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override*.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+; override.addr
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+
+; override.addr.dim (1D only)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i1 %flag_ch)
+
+; override.addr.dim.stride (2D and higher)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i64 %ch, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+
+; scatter4 + override.addr (2D only)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 %flag_ch)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override*.[1-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.[1-5]d.*` set of PTX instructions qualified with the `.override::*` qualifiers.
+These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `tile` mode, while overriding specific properties of the opaque tensor-map object with explicit instruction operands.
+
+The `.override::*` qualifiers allow certain tensor-map properties to be ignored and instead supplied as explicit operands.
+The supported override variants are:
+
+- `override.addr` (`.override::global_address`): The global base address from the tensor-map is ignored and the explicit `ptr addrspace(1) %override_addr` operand is used instead.
+ The address must be 16B aligned; otherwise a runtime error is raised.
+ The memory range `[%override_addr, %override_addr + 128 KiB)` must be allocated and accessible during execution; otherwise the behavior is undefined.
+
+- `override.addr.dim` (`.override::global_address` + `.override::global_dim`, 1D only): Overrides the base address and the tensor global dimension.
+ The `i16 %ts0` operand specifies the new tensor size.
+ This variant requires the `tile` load mode; the base address override is mandatory when overriding attributes.
+ The tensor start coordinates must be zero; otherwise the behavior is undefined.
+
+- `override.addr.dim.stride` (`.override::global_address` + `.override::global_dim_stride`, 2D and higher): Overrides the base address, the tensor global dimensions, and the global strides.
+ The `i16 %ts0 ... i16 %ts{dim-1}` operands specify the new tensor sizes, the `i32 %stride0 ... i32 %stride{dim-2}` operands specify the lower strides, and the `i16 %upper_stride` operand specifies the upper stride.
+ The effective global stride is computed as `global_stride[i] = ((%stride{i} + (%upper_stride{i} << 32)) << 4)`.
+ This variant requires the `tile` load mode; the base address override is mandatory when overriding attributes.
+ The tensor start coordinates must be zero; otherwise the behavior is undefined.
+
+The `scatter4.override.addr` variant combines the `tile.scatter4` mode (2D only) with the base address override.
+In `tile.scatter4` mode, a single 2D source tensor is divided into four rows in the 2D destination tensor.
+The first coordinate `i32 %x0` denotes the column index followed by four coordinates indicating the four row-indices.
+For more information on `scatter4` mode, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-tiled-scatter4-gather4-modes).
+
+- The last argument to these intrinsics is a boolean flag indicating support for cache_hint.
+ This flag argument must be a compile-time constant.
+ When set, it indicates a valid cache_hint (`i64 %ch`) and generates the `.L2::cache_hint` variant of the PTX instruction.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.[3-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.im2col_no_offs.[3-5]d.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
+These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `im2col` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
+The address must be 16B aligned and the memory range `[%override_addr, %override_addr + 128 KiB)` must be allocated and accessible during execution; otherwise the behavior is undefined.
+The override address semantics and the last boolean flag argument have the same functionality as described in the `s2g.tile` override intrinsics above.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.[3-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.im2col_no_offs::w.[3-5]d.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
+These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `im2col_w` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
+The override address semantics and the last boolean flag argument have the same functionality as described in the `s2g.tile` override intrinsics above.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override*.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+; override.addr
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 %red_op, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+
+; override.addr.dim (1D only)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 %red_op, i1 %flag_ch)
+
+; override.addr.dim.stride (2D and higher)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i64 %ch, i32 %red_op, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override*.[1-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.[1-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::*` qualifiers.
+These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `tile` mode, while overriding specific properties of the opaque tensor-map object with explicit instruction operands.
+The three override variants (`override.addr`, `override.addr.dim`, and `override.addr.dim.stride`) have the same semantics as described in the `s2g.tile` override intrinsics above.
+
+The `i32 %red_op` argument selects the reduction operation to perform.
+It must be a compile-time constant in the half-open range `[0, 8)`, with the following encoding:
+
+| `red_op` | Reduction Operation |
+|:--------:|:--------------------|
+| 0 | `add` |
+| 1 | `min` |
+| 2 | `max` |
+| 3 | `inc` |
+| 4 | `dec` |
+| 5 | `and` |
+| 6 | `or` |
+| 7 | `xor` |
+
+The symbolic LLVM IR annotation for `red_op` and the PTX reduction suffix use the same canonical operator spelling.
+
+- The last argument to these intrinsics is a boolean flag indicating support for cache_hint.
+ This flag argument must be a compile-time constant.
+ When set, it indicates a valid cache_hint (`i64 %ch`) and generates the `.L2::cache_hint` variant of the PTX instruction.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 %red_op, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.[3-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.im2col_no_offs.[3-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
+These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `im2col` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
+The `i32 %red_op` argument, the override address semantics, and the last boolean flag argument have the same functionality as described in the `reduce.tile` override intrinsics above.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 %red_op, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.[3-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.im2col_no_offs::w.[3-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
+These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `im2col_w` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
+The `i32 %red_op` argument, the override address semantics, and the last boolean flag argument have the same functionality as described in the `reduce.tile` override intrinsics above.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
+
### Warp Group Intrinsics
#### '`llvm.nvvm.wgmma.fence.sync.aligned`'
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index e6761a852c93f..2339eed36b54a 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1268,6 +1268,93 @@ class NVVMPureIntrinsic<list<LLVMType> ret_types,
DefaultAttrsIntrinsic<ret_types, param_types,
intr_properties # [IntrNoMem, IntrSpeculatable], name> {}
+// Class to handle TMA Copy Override Support
+class TMAOverrideHandler<int dim, string mode> {
+ // TMA Override Support:
+ // +----------------+------------------+-------------------------+
+ // | Mode | Dimensions | Supported Overrides |
+ // +----------------+------------------+-------------------------+
+ // | Tile | 1D | override_addr |
+ // | | | override_addr_dim |
+ // +----------------+------------------+-------------------------+
+ // | Tile | 2D and higher | override_addr |
+ // | | | override_addr_dim_stride|
+ // +----------------+------------------+-------------------------+
+ // | Im2col | 3D and higher | override_addr |
+ // +----------------+------------------+-------------------------+
+ list<string> common_overrides = ["override_addr"];
+ list<string> tile_overrides_1d = !listconcat(common_overrides, ["override_addr_dim"]);
+ list<string> tile_overrides_nd = !listconcat(common_overrides, ["override_addr_dim_stride"]);
+
+ // Return the applicable overrides based on mode and dimension
+ list<string> applicable_overrides = !if(!eq(mode, "tile"),
+ !if(!eq(dim, 1), tile_overrides_1d, tile_overrides_nd),
+ common_overrides);
+}
+
+class CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
+ list<LLVMType> OverrideTy = [llvm_global_ptr_ty];
+ list<LLVMType> TensorSizeTy = !listsplat(llvm_i16_ty, dim);
+ list<LLVMType> LowerStrideTy = !listsplat(llvm_i32_ty, !add(dim, -1));
+ list<LLVMType> UpperStrideTy = [llvm_i16_ty];
+ list<LLVMType> OverrideDimTy = !listconcat(OverrideTy, TensorSizeTy);
+ list<LLVMType> OverrideDimStrideTy = !listconcat(OverrideTy, TensorSizeTy, LowerStrideTy, UpperStrideTy);
+ list<LLVMType> OverrideAddrTy = !cond(!eq(override, "override_addr"): OverrideTy,
+ !eq(override, "override_addr_dim"): OverrideDimTy,
+ !eq(override, "override_addr_dim_stride"): OverrideDimStrideTy);
+ list<LLVMType> TensorDimsTy = !listsplat(llvm_i32_ty, dim);
+
+ // For im2col_w/w128 mode, NumIm2ColOffsets is always 2 irrespective of the
+ // dim (wHalo and wOffset)
+ int NumIm2ColOffsets =
+ !cond(!eq(mode, "im2col_w") : 2,
+ !eq(mode, "im2col_w_128") : 2,
+ !eq(mode, "im2col") : !add(dim, -2),
+ true : 0);
+ list<LLVMType> Im2ColOffsetsTy = !listsplat(llvm_i16_ty, NumIm2ColOffsets);
+
+ string Prefix = "int_nvvm_cp_async_bulk_tensor_";
+ string Suffix = "_" # mode # "_" # override # "_" # dim # "d";
+}
+
+class CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode, string override>
+ : CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<dim, mode, override> {
+ string Name = Prefix # "s2g" # Suffix;
+
+ list<LLVMType> ArgsTy = !listconcat(
+ [llvm_shared_ptr_ty, // src_smem_ptr
+ llvm_ptr_ty], // tensormap_ptr
+ OverrideAddrTy, // override_addr
+ TensorDimsTy, // actual tensor dims
+ [llvm_i64_ty] // cache_hint
+ );
+ list<LLVMType> FlagsTy = [llvm_i1_ty];
+ defvar FlagCacheHintIdx = !size(ArgsTy);
+ list<IntrinsicProperty> IntrProp = [
+ IntrConvergent,
+ ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>, WriteOnly<ArgIndex<2>>,
+ ArgInfo<ArgIndex<FlagCacheHintIdx>, [ArgName<"flag_cache_hint">]>
+ ];
+}
+
+class CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode, string override>
+ : CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<dim, mode, override> {
+ string Name = Prefix # "reduce" # Suffix;
+
+ defvar RedOpIdx = !size(ArgsTy);
+ list<LLVMType> FlagsTy = [llvm_i32_ty, // reduction operation
+ llvm_i1_ty]; // Flag for cache_hint
+ list<IntrinsicProperty> IntrProp = [
+ IntrConvergent,
+ ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
+ // Allowed values for red_op are {0..7} i.e. [0, 8).
+ Range<ArgIndex<RedOpIdx>, 0, 8>,
+ ArgInfo<ArgIndex<RedOpIdx>, [ArgName<"red_op">,
+ ImmArgPrinter<"printTMAReductionOp">]>,
+ ArgInfo<ArgIndex<!add(RedOpIdx, 1)>, [ArgName<"flag_cache_hint">]>
+ ];
+}
+
let TargetPrefix = "nvvm" in {
//
@@ -2874,6 +2961,42 @@ def int_nvvm_cp_async_bulk_tensor_s2g_tile_scatter4_2d
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>]>;
+// TMA Tensor Copy Intrinsics: S2G -> From shared to global memory with override address variants
+foreach dim = 1...5 in {
+ foreach mode = !if(!ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
+ foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
+ // Reduce + Override
+ foreach reduce_override =
+ [CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<dim, mode, override>] in
+ def reduce_override.Name
+ : DefaultAttrsIntrinsicFlags<[], reduce_override.ArgsTy,
+ reduce_override.FlagsTy,
+ reduce_override.IntrProp>;
+
+ // S2G + Override
+ foreach s2g_override =
+ [CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<dim, mode, override>] in
+ def s2g_override.Name
+ : DefaultAttrsIntrinsicFlags<[], s2g_override.ArgsTy,
+ s2g_override.FlagsTy,
+ s2g_override.IntrProp>;
+ }
+ }
+}
+
+// S2G + Override + tile::scatter4
+def int_nvvm_cp_async_bulk_tensor_s2g_tile_scatter4_override_addr_2d
+ : DefaultAttrsIntrinsicFlags<[],
+ !listconcat([llvm_shared_ptr_ty, // src_smem_ptr
+ llvm_ptr_ty, // tensormap_ptr
+ llvm_global_ptr_ty], // override_addr
+ !listsplat(llvm_i32_ty, 5), // dims
+ [llvm_i64_ty]), // cache_hint
+ [llvm_i1_ty], // Flag for cache_hint
+ [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
+ WriteOnly<ArgIndex<2>>, ArgInfo<ArgIndex<9>,
+ [ArgName<"flag_cache_hint">]>]>;
+
// TMA Tensor Copy Intrinsics: G2S -> From Global to Shared memory variants
foreach dim = 1...5 in {
defvar tensor_dim_args = !listsplat(llvm_i32_ty, dim);
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 72b614d84e3e3..09b7901010ffa 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -858,6 +858,105 @@ foreach dim = 3...5 in {
defm TMA_S2G_TILE_SCATTER4_2D : TMA_TENSOR_S2G_INTR<5, "tile_scatter4",
[hasTMABlackwellSupport]>;
+// Base class for TMA tensor override intrinsics
+class TMA_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
+ // Dimension handling
+ dag dims_dag = !dag(ins, !listsplat(B32, dim), !foreach(i, !range(dim), "d" # i));
+ string dims_str = !interleave(!foreach(i, !range(dim), "$d" # i), ", ");
+ bit has_multi_dim = !ge(dim, 2);
+
+ // Im2col handling (im2col / im2col_w / im2col_w_128)
+ int num_im2col = !cond(!eq(mode, "im2col"): !add(dim, -2),
+ !eq(mode, "im2col_w"): 2,
+ !eq(mode, "im2col_w_128"): 2,
+ true: 0);
+ dag im2col_dag = !dag(ins, !listsplat(B16, num_im2col),
+ !foreach(i, !range(num_im2col), "im2col" # i));
+ string im2col_str = !interleave(!foreach(i, !range(num_im2col), "$im2col" # i), ", ");
+ string im2col_component = !if(num_im2col, ", {{" # im2col_str # "}}", "");
+
+ // Tensor size handling
+ bit is_addr_dim_stride = !eq(override, "override_addr_dim_stride");
+ dag tensor_size_dag = !if(is_addr_dim_stride,
+ !dag(ins, !listsplat(B16, dim), !foreach(i, !range(dim), "ts" # i)),
+ (ins));
+ string tensor_size_str = !interleave(!foreach(i, !range(dim), "$ts" # i), ", ");
+
+ // Stride handling for multi-dimensional tensors
+ bit need_strides = !and(is_addr_dim_stride, has_multi_dim);
+ dag lower_stride_dag = !if(need_strides,
+ !dag(ins, !listsplat(B32, !sub(dim, 1)),
+ !foreach(i, !range(!sub(dim, 1)), "stride" # i)),
+ (ins));
+ string lower_stride_str = !interleave(!foreach(i, !range(!sub(dim, 1)), "$stride" # i), ", ");
+ dag upper_stride_dag = !if(need_strides, (ins B16:$upper_stride), (ins));
+
+ // Compose assembly string components
+ string stride_component = !if(has_multi_dim,
+ "{{" # lower_stride_str # "}}, $upper_stride, ",
+ "");
+ string tensor_size_component = !if(is_addr_dim_stride,
+ "{{" # tensor_size_str # "}}, " # stride_component,
+ "");
+
+ // Common suffix components
+ string dim_suffix = !if(has_multi_dim,
+ ".override::global_dim_stride",
+ ".override::global_dim");
+ string override_suffix = ".override::global_address" # !if(is_addr_dim_stride, dim_suffix, "");
+}
+
+// TMA Copy from Shared to Global memory (with Override Global Address)
+multiclass NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode,
+ string override> {
+ defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
+ defvar is_scatter4 = !eq(mode, "tile_scatter4");
+ defvar mode_asm_str = !cond(
+ !eq(mode, "im2col") : "im2col_no_offs",
+ !eq(mode, "im2col_w") : "im2col_no_offs::w",
+ !eq(mode, "tile_scatter4") : "tile::scatter4",
+ true : mode);
+ defvar inst_name = "cp.async.bulk.tensor." # !if(is_scatter4, 2, dim) #
+ "d.global.shared::cta." # mode_asm_str # ".bulk_group";
+ defvar suffix = base.override_suffix;
+ defvar asm_operands = " [$tmap, $override_addr, " # base.tensor_size_component #
+ "{{" # base.dims_str # "}}], [$src]";
+
+ defvar is_override_dim_stride_1d =
+ !and(base.is_addr_dim_stride, !eq(dim, 1));
+ defvar intr = !cast<Intrinsic>(
+ "int_nvvm_cp_async_bulk_tensor_s2g_" # mode # "_" #
+ !if(is_override_dim_stride_1d, "override_addr_dim", override) # "_" #
+ !if(is_scatter4, 2, dim) # "d");
+
+ defvar ins_dag_base =
+ !con((ins ADDR:$src, B64:$tmap, B64:$override_addr),
+ base.tensor_size_dag, base.lower_stride_dag, base.upper_stride_dag,
+ base.dims_dag);
+ defvar ins_dag = ins_dag_base;
+ defvar ins_dag_ch = !con(ins_dag_base, (ins B64:$ch));
+
+ defvar intr_dag_base =
+ !con((intr addr:$src, B64:$tmap, B64:$override_addr),
+ !setdagop(base.tensor_size_dag, intr),
+ !setdagop(base.lower_stride_dag, intr),
+ !setdagop(base.upper_stride_dag, intr),
+ !setdagop(base.dims_dag, intr));
+ defvar intr_dag =
+ !con(intr_dag_base, (intr (i64 srcvalue), 0));
+ defvar intr_dag_ch =
+ !con(intr_dag_base, (intr i64:$ch, -1));
+
+ let Predicates = [callSubtarget<"hasRubinFamilySupport">] in {
+ def "" : NVPTXInst<(outs), ins_dag,
+ inst_name # suffix # asm_operands # ";",
+ [intr_dag]>;
+ def _CH : NVPTXInst<(outs), ins_dag_ch,
+ inst_name # suffix # ".L2::cache_hint" # asm_operands # ", $ch;",
+ [intr_dag_ch]>;
+ }
+}
+
def TMAReductionFlags : Operand<i32> {
let PrintMethod = "printTmaReductionMode";
}
@@ -865,6 +964,81 @@ def TMAReductionFlags : Operand<i32> {
def tma_tensor_reduction_imm :
TImmLeaf<i32, [{ return Imm >= 0 && Imm < 8; }]>;
+// TMA Copy from Shared to Global memory with Reduction (with Override Global Address)
+multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
+ string override> {
+ defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
+ defvar mode_asm_str = !cond(!eq(mode, "im2col") : "im2col_no_offs",
+ !eq(mode, "im2col_w") : "im2col_no_offs::w",
+ true : mode);
+ defvar inst_name = "cp.reduce.async.bulk.tensor." # dim #
+ "d.global.shared::cta";
+ defvar suffix = "." # mode_asm_str # ".bulk_group" # base.override_suffix;
+ defvar asm_operands = " [$tmap, $override_addr, " # base.tensor_size_component #
+ "{{" # base.dims_str # "}}], [$src]";
+
+ defvar is_override_dim_stride_1d =
+ !and(base.is_addr_dim_stride, !eq(dim, 1));
+
+ defvar intr = !cast<Intrinsic>(
+ "int_nvvm_cp_async_bulk_tensor_reduce_" # mode # "_" #
+ !if(is_override_dim_stride_1d, "override_addr_dim", override) # "_" #
+ dim # "d");
+
+ defvar ins_dag_base =
+ !con((ins ADDR:$src, B64:$tmap, B64:$override_addr),
+ base.tensor_size_dag, base.lower_stride_dag, base.upper_stride_dag,
+ base.dims_dag);
+ defvar ins_dag = !con(ins_dag_base, (ins TMAReductionFlags:$red_op));
+ defvar ins_dag_ch =
+ !con(ins_dag_base, (ins B64:$ch, TMAReductionFlags:$red_op));
+
+ defvar intr_dag_base =
+ !con((intr addr:$src, B64:$tmap, B64:$override_addr),
+ !setdagop(base.tensor_size_dag, intr),
+ !setdagop(base.lower_stride_dag, intr),
+ !setdagop(base.upper_stride_dag, intr),
+ !setdagop(base.dims_dag, intr));
+ defvar intr_dag =
+ !con(intr_dag_base,
+ (intr (i64 srcvalue), tma_tensor_reduction_imm:$red_op, 0));
+ defvar intr_dag_ch =
+ !con(intr_dag_base,
+ (intr B64:$ch, tma_tensor_reduction_imm:$red_op, -1));
+
+ let Predicates = [callSubtarget<"hasRubinFamilySupport">] in {
+ def "" : NVPTXInst<(outs), ins_dag,
+ inst_name # "${red_op}" # suffix # asm_operands # ";",
+ [intr_dag]>;
+ def _CH : NVPTXInst<(outs), ins_dag_ch,
+ inst_name # "${red_op}" # suffix # ".L2::cache_hint" # asm_operands #
+ ", $ch;",
+ [intr_dag_ch]>;
+ }
+}
+
+foreach dim = 1...5 in {
+ foreach mode = !if(
+ !ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
+ foreach override = !if(!eq(mode, "tile"),
+ ["override_addr", "override_addr_dim_stride"],
+ ["override_addr"]) in {
+ defm S2G_ # dim # "D_" #
+ !toupper(!subst("::", "_", mode) # "_" # override)
+ : NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<
+ dim, mode, override>;
+ defm RED_ # dim # "D_" #
+ !toupper(!subst("::", "_", mode) # "_" # override)
+ : NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<
+ dim, mode, override>;
+ }
+ }
+}
+
+defm S2G_2D_TILE_SCATTER4_OVERRIDE_ADDR
+ : NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<
+ 5, "tile_scatter4", "override_addr">;
+
// TMA Copy from Shared to Global memory with Reduction
multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, string mode,
list<Predicate> pred = [PTX80, SM90]> {
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll
new file mode 100644
index 0000000000000..b8790d436d71f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94| %ptxas-verify -arch=sm_94 %}
+
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=CHECK-FORMAT
+
+define void @cp_async_bulk_tensor_reduce_tile_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_override_addr_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_override_addr_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_override_addr_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_override_addr_param_3];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_override_addr_param_4];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_override_addr_param_5];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_override_addr_param_6];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_tile_override_addr_param_7];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_tile_override_addr_param_8];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+
+ ; Test min reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 1, i1 1)
+
+ ; Test max reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 2, i1 1)
+
+ ; Test inc reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 3, i1 1)
+
+ ; Test dec reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 4, i1 1)
+
+ ; Test and reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 5, i1 1)
+
+ ; Test or reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 6, i1 1)
+
+ ; Test xor reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 7, i1 1)
+
+ ret void
+}
+
+define void @cp_async_bulk_tensor_reduce_im2col_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_5];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_6];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_7];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_8];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_im2col_override_addr(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i32 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+
+ ; Test min reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
+
+ ; Test max reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
+
+ ; Test inc reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
+
+ ; Test dec reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
+
+ ; Test and reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
+
+ ; Test or reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
+
+ ; Test xor reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
+
+ ret void
+}
+
+define void @cp_async_bulk_tensor_reduce_im2col_w_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_w_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_5];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_6];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_7];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_8];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_im2col_w_override_addr(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i32 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+
+ ; Test min reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
+
+ ; Test max reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
+
+ ; Test inc reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
+
+ ; Test dec reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
+
+ ; Test and reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
+
+ ; Test or reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
+
+ ; Test xor reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
+
+ ret void
+}
+
+define void @cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param.b16 %rs1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param.b16 %rs3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs5, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param.b32 %r6, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param.b32 %r7, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs6, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param.b32 %r8, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param.b32 %r9, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_17];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_18];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+
+ ; Test min reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 1, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 1, i1 1)
+
+ ; Test max reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 2, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 2, i1 1)
+
+ ; Test inc reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 3, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 3, i1 1)
+
+ ; Test dec reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 4, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 4, i1 1)
+
+ ; Test and reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 5, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 5, i1 1)
+
+ ; Test or reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 6, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 6, i1 1)
+
+ ; Test xor reduction operation
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 7, i1 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 7, i1 1)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll
new file mode 100644
index 0000000000000..d33d1a52c12c6
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll
@@ -0,0 +1,247 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94| %ptxas-verify -arch=sm_94 %}
+
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=CHECK-FORMAT
+
+define void @cp_async_bulk_tensor_s2g_tile_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_override_addr_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_override_addr_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_override_addr_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_tile_override_addr_param_3];
+; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_tile_override_addr_param_4];
+; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_tile_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_tile_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_tile_override_addr_param_7];
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ ret void
+}
+
+define void @cp_async_bulk_tensor_s2g_im2col_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_s2g_im2col_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_7];
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_im2col_override_addr(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ ret void
+}
+
+define void @cp_async_bulk_tensor_s2g_im2col_w_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_s2g_im2col_w_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_7];
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_im2col_w_override_addr(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ ret void
+}
+
+define void @cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param.b16 %rs1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param.b16 %rs3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs5, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param.b32 %r6, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param.b32 %r7, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1];
+; CHECK-NEXT: ld.param.b16 %rs6, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param.b32 %r8, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param.b32 %r9, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_17];
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_18];
+; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1], %rd4;
+; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ ret void
+}
+
+define void @cp_async_bulk_tensor_s2g_tile_scatter4_2d(i32 %flag, ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_s2g_tile_scatter4_2d(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_1];
+; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_2];
+; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_3];
+; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_4];
+; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_5];
+; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_8];
+; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile::scatter4.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_9];
+; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile::scatter4.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_tile_scatter4_2d(
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+
+ ret void
+}
>From 67fc2456420d58dd7a737a9e352a07964e86be0c Mon Sep 17 00:00:00 2001
From: Rajat Bajpai <rbajpai at nvidia.com>
Date: Wed, 12 Aug 2026 09:12:23 +0000
Subject: [PATCH 2/3] Addressed review comments
---
llvm/include/llvm/IR/IntrinsicsNVVM.td | 41 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 62 +--
.../cp-async-bulk-tensor-reduce-override.ll | 512 +++++++++---------
.../cp-async-bulk-tensor-s2g-override.ll | 250 +++++----
4 files changed, 427 insertions(+), 438 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 2339eed36b54a..109b8cd171e51 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1268,6 +1268,18 @@ class NVVMPureIntrinsic<list<LLVMType> ret_types,
DefaultAttrsIntrinsic<ret_types, param_types,
intr_properties # [IntrNoMem, IntrSpeculatable], name> {}
+// For im2col mode the number of offsets is (dim - 2).
+// For im2col_w / im2col_w_128 modes it is always 2 (wHalo and wOffset),
+// irrespective of dim.
+// For all other modes (e.g. tile) there are no offsets.
+class TMA_IM2COL_OFFSETS_UTIL<int dim, string mode> {
+ int num = !cond(
+ !eq(mode, "im2col") : !sub(dim, 2),
+ !eq(mode, "im2col_w") : 2,
+ !eq(mode, "im2col_w_128"): 2,
+ true : 0);
+}
+
// Class to handle TMA Copy Override Support
class TMAOverrideHandler<int dim, string mode> {
// TMA Override Support:
@@ -1293,24 +1305,18 @@ class TMAOverrideHandler<int dim, string mode> {
}
class CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
- list<LLVMType> OverrideTy = [llvm_global_ptr_ty];
+ list<LLVMType> AddrTy = [llvm_global_ptr_ty];
list<LLVMType> TensorSizeTy = !listsplat(llvm_i16_ty, dim);
list<LLVMType> LowerStrideTy = !listsplat(llvm_i32_ty, !add(dim, -1));
list<LLVMType> UpperStrideTy = [llvm_i16_ty];
- list<LLVMType> OverrideDimTy = !listconcat(OverrideTy, TensorSizeTy);
- list<LLVMType> OverrideDimStrideTy = !listconcat(OverrideTy, TensorSizeTy, LowerStrideTy, UpperStrideTy);
- list<LLVMType> OverrideAddrTy = !cond(!eq(override, "override_addr"): OverrideTy,
- !eq(override, "override_addr_dim"): OverrideDimTy,
- !eq(override, "override_addr_dim_stride"): OverrideDimStrideTy);
+ list<LLVMType> AddrDimTy = !listconcat(AddrTy, TensorSizeTy);
+ list<LLVMType> AddrDimStrideTy = !listconcat(AddrTy, TensorSizeTy, LowerStrideTy, UpperStrideTy);
+ list<LLVMType> OverrideTypes = !cond(!eq(override, "override_addr"): AddrTy,
+ !eq(override, "override_addr_dim"): AddrDimTy,
+ !eq(override, "override_addr_dim_stride"): AddrDimStrideTy);
list<LLVMType> TensorDimsTy = !listsplat(llvm_i32_ty, dim);
- // For im2col_w/w128 mode, NumIm2ColOffsets is always 2 irrespective of the
- // dim (wHalo and wOffset)
- int NumIm2ColOffsets =
- !cond(!eq(mode, "im2col_w") : 2,
- !eq(mode, "im2col_w_128") : 2,
- !eq(mode, "im2col") : !add(dim, -2),
- true : 0);
+ int NumIm2ColOffsets = TMA_IM2COL_OFFSETS_UTIL<dim, mode>.num;
list<LLVMType> Im2ColOffsetsTy = !listsplat(llvm_i16_ty, NumIm2ColOffsets);
string Prefix = "int_nvvm_cp_async_bulk_tensor_";
@@ -1324,7 +1330,7 @@ class CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode, string overri
list<LLVMType> ArgsTy = !listconcat(
[llvm_shared_ptr_ty, // src_smem_ptr
llvm_ptr_ty], // tensormap_ptr
- OverrideAddrTy, // override_addr
+ OverrideTypes, // override_addr (dst)
TensorDimsTy, // actual tensor dims
[llvm_i64_ty] // cache_hint
);
@@ -2961,7 +2967,8 @@ def int_nvvm_cp_async_bulk_tensor_s2g_tile_scatter4_2d
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>]>;
-// TMA Tensor Copy Intrinsics: S2G -> From shared to global memory with override address variants
+// TMA Tensor Copy/Reduction Intrinsics: S2G -> From shared to global memory
+// with override address variants
foreach dim = 1...5 in {
foreach mode = !if(!ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
@@ -3005,9 +3012,7 @@ foreach dim = 1...5 in {
defvar is_im2col = !eq(mode, "im2col");
defvar is_im2colw = !or(!eq(mode, "im2col_w"), !eq(mode, "im2col_w_128"));
- // For im2col_w/w128 modes, the num_offsets is always 2.
- // For im2col mode, the num_offsets is (dim - 2).
- defvar num_im2col_offsets = !if(is_im2colw, 2, !if(is_im2col, !add(dim, -2), 0));
+ defvar num_im2col_offsets = TMA_IM2COL_OFFSETS_UTIL<dim, mode>.num;
defvar im2col_offsets_args = !listsplat(llvm_i16_ty, num_im2col_offsets);
defvar g2s_params = !listconcat(
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 09b7901010ffa..338cdd2b4d13b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -642,14 +642,7 @@ class TMA_DIMS_UTIL<int dim, string mode = ""> {
}
class TMA_IM2COL_UTIL<int dim, string mode> {
- // For im2col_w/w_128 modes, number of offsets is always 2.
- // For im2col mode, offsets is (dim - 2).
- // For non-im2col modes (i.e. tile) there are no offsets.
- int offsets = !cond(
- !eq(mode, "im2col") : !sub(dim, 2),
- !eq(mode, "im2col_w") : 2,
- !eq(mode, "im2col_w_128") : 2,
- true : 0); // for all other modes
+ int offsets = TMA_IM2COL_OFFSETS_UTIL<dim, mode>.num;
dag ins_dag = !if(!gt(offsets, 0),
!dag(ins, !listsplat(B16, offsets), !foreach(i, !range(offsets), "im2col" # i)),
@@ -657,6 +650,19 @@ class TMA_IM2COL_UTIL<int dim, string mode> {
string base_str = !interleave(!foreach(i, !range(offsets), "$im2col" # i), ", ");
}
+// Maps a TMA tensor copy "mode" to equivalent assembly mnemonic.
+// tile -> tile
+// im2col -> im2col_no_offs
+// im2col_w -> im2col_no_offs::w
+// tile_scatter4 -> tile::scatter4
+class TMA_MODE_ASM_UTIL<string mode> {
+ string ret = !cond(
+ !eq(mode, "im2col") : "im2col_no_offs",
+ !eq(mode, "im2col_w") : "im2col_no_offs::w",
+ !eq(mode, "tile_scatter4") : "tile::scatter4",
+ true : mode);
+}
+
def CTAGroupFlags : Operand<i32> {
let PrintMethod = "printCTAGroup";
}
@@ -822,11 +828,7 @@ multiclass TMA_TENSOR_S2G_INTR<int dim, string mode,
(intr B64:$ch, -1));
// Fix-up the asm_str when it is im2col/scatter4.
- defvar mode_asm_str = !cond(
- !eq(mode, "im2col") : "im2col_no_offs",
- !eq(mode, "im2col_w") : "im2col_no_offs::w",
- !eq(mode, "tile_scatter4") : "tile::scatter4",
- true : mode);
+ defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
defvar prefix = "cp.async.bulk.tensor"
# "." # dim_val # "d"
# ".global.shared::cta"
@@ -860,19 +862,17 @@ defm TMA_S2G_TILE_SCATTER4_2D : TMA_TENSOR_S2G_INTR<5, "tile_scatter4",
// Base class for TMA tensor override intrinsics
class TMA_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
- // Dimension handling
- dag dims_dag = !dag(ins, !listsplat(B32, dim), !foreach(i, !range(dim), "d" # i));
- string dims_str = !interleave(!foreach(i, !range(dim), "$d" # i), ", ");
+ // Dimension handling.
+ defvar dims_util = TMA_DIMS_UTIL<dim>;
+ dag dims_dag = dims_util.ins_dag;
+ string dims_str = dims_util.base_str;
bit has_multi_dim = !ge(dim, 2);
- // Im2col handling (im2col / im2col_w / im2col_w_128)
- int num_im2col = !cond(!eq(mode, "im2col"): !add(dim, -2),
- !eq(mode, "im2col_w"): 2,
- !eq(mode, "im2col_w_128"): 2,
- true: 0);
- dag im2col_dag = !dag(ins, !listsplat(B16, num_im2col),
- !foreach(i, !range(num_im2col), "im2col" # i));
- string im2col_str = !interleave(!foreach(i, !range(num_im2col), "$im2col" # i), ", ");
+ // Im2col handling (im2col / im2col_w / im2col_w_128).
+ defvar im2col_util = TMA_IM2COL_UTIL<dim, mode>;
+ int num_im2col = im2col_util.offsets;
+ dag im2col_dag = im2col_util.ins_dag;
+ string im2col_str = im2col_util.base_str;
string im2col_component = !if(num_im2col, ", {{" # im2col_str # "}}", "");
// Tensor size handling
@@ -911,11 +911,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode,
string override> {
defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
defvar is_scatter4 = !eq(mode, "tile_scatter4");
- defvar mode_asm_str = !cond(
- !eq(mode, "im2col") : "im2col_no_offs",
- !eq(mode, "im2col_w") : "im2col_no_offs::w",
- !eq(mode, "tile_scatter4") : "tile::scatter4",
- true : mode);
+ defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
defvar inst_name = "cp.async.bulk.tensor." # !if(is_scatter4, 2, dim) #
"d.global.shared::cta." # mode_asm_str # ".bulk_group";
defvar suffix = base.override_suffix;
@@ -968,9 +964,7 @@ def tma_tensor_reduction_imm :
multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
string override> {
defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
- defvar mode_asm_str = !cond(!eq(mode, "im2col") : "im2col_no_offs",
- !eq(mode, "im2col_w") : "im2col_no_offs::w",
- true : mode);
+ defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
defvar inst_name = "cp.reduce.async.bulk.tensor." # dim #
"d.global.shared::cta";
defvar suffix = "." # mode_asm_str # ".bulk_group" # base.override_suffix;
@@ -1046,9 +1040,7 @@ multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, string mode,
defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
- defvar mode_asm_str = !cond(!eq(mode, "im2col") : "im2col_no_offs",
- !eq(mode, "im2col_w") : "im2col_no_offs::w",
- true : mode);
+ defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
defvar prefix = "cp.reduce.async.bulk.tensor"
# "." # dim # "d"
# ".global.shared::cta";
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll
index b8790d436d71f..8707d005b02d6 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce-override.ll
@@ -4,27 +4,27 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=CHECK-FORMAT
-define void @cp_async_bulk_tensor_reduce_tile_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_reduce_tile_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_override_addr_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_override_addr_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_override_addr_param_2];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_reduce_tile_override_addr_param_3];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_reduce_tile_override_addr_param_4];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_reduce_tile_override_addr_param_5];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_reduce_tile_override_addr_param_6];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_tile_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_reduce_tile_override_addr_param_7];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_tile_override_addr_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_reduce_tile_override_addr_param_8];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
@@ -45,105 +45,104 @@ define void @cp_async_bulk_tensor_reduce_tile_override_addr(ptr addrspace(3) %sr
; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
; Test min reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 1, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 1, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 1, i1 1)
; Test max reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 2, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 2, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 2, i1 1)
; Test inc reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 3, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 3, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 3, i1 1)
; Test dec reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 4, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 4, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 4, i1 1)
; Test and reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 5, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 5, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 5, i1 1)
; Test or reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 6, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 6, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 6, i1 1)
; Test xor reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 7, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i32 7, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i32 7, i1 1)
ret void
}
-define void @cp_async_bulk_tensor_reduce_im2col_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_reduce_im2col_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_override_addr(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_2];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_5];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_6];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_7];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_reduce_im2col_override_addr_param_8];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
@@ -162,93 +161,92 @@ define void @cp_async_bulk_tensor_reduce_im2col_override_addr(ptr addrspace(3) %
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_im2col_override_addr(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i32 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
; Test min reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 1, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
; Test max reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 2, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
; Test inc reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 3, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
; Test dec reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 4, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
; Test and reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 5, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
; Test or reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 6, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
; Test xor reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 7, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
ret void
}
-define void @cp_async_bulk_tensor_reduce_im2col_w_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_reduce_im2col_w_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_w_override_addr(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_2];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_5];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_6];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_7];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_reduce_im2col_w_override_addr_param_8];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
@@ -267,75 +265,74 @@ define void @cp_async_bulk_tensor_reduce_im2col_w_override_addr(ptr addrspace(3)
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_im2col_w_override_addr(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i32 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
; Test min reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 1, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
; Test max reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 2, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
; Test inc reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 3, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
; Test dec reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 4, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
; Test and reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 5, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
; Test or reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 6, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
; Test xor reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 7, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
ret void
}
-define void @cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<7>;
@@ -343,30 +340,30 @@ define void @cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(ptr addrs
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_2];
-; CHECK-NEXT: ld.param.b16 %rs1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_3];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_13];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_4];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_8];
-; CHECK-NEXT: ld.param.b16 %rs3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_12];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_14];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_5];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_9];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: ld.param::func.b16 %rs4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_15];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs5, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_6];
-; CHECK-NEXT: ld.param.b32 %r6, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_10];
-; CHECK-NEXT: ld.param.b32 %r7, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: ld.param::func.b16 %rs5, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r6, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param::func.b32 %r7, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_16];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs6, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_7];
-; CHECK-NEXT: ld.param.b32 %r8, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_11];
-; CHECK-NEXT: ld.param.b32 %r9, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_17];
+; CHECK-NEXT: ld.param::func.b16 %rs6, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r8, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param::func.b32 %r9, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_17];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_18];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride_param_18];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1], %rd4;
@@ -401,116 +398,115 @@ define void @cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(ptr addrs
; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
; CHECK-NEXT: cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_reduce_tile_override_addr_dim_stride(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=min */ i32 1, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=max */ i32 2, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=inc */ i32 3, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=dec */ i32 4, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=and */ i32 5, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=or */ i32 6, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* red_op=xor */ i32 7, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
; Test min reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 1, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 1, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 1, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 1, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 1, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 1, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 1, i1 1)
; Test max reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 2, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 2, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 2, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 2, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 2, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 2, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 2, i1 1)
; Test inc reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 3, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 3, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 3, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 3, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 3, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 3, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 3, i1 1)
; Test dec reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 4, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 4, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 4, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 4, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 4, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 4, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 4, i1 1)
; Test and reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 5, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 5, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 5, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 5, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 5, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 5, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 5, i1 1)
; Test or reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 6, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 6, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 6, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 6, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 6, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 6, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 6, i1 1)
; Test xor reduction operation
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i32 7, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i32 7, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i32 7, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 7, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 7, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i32 7, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i32 7, i1 1)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll
index d33d1a52c12c6..ce429dc279cdd 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g-override.ll
@@ -5,151 +5,149 @@
; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=CHECK-FORMAT
-define void @cp_async_bulk_tensor_s2g_tile_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_s2g_tile_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_override_addr_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_override_addr_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_override_addr_param_2];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_tile_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_s2g_tile_override_addr_param_3];
; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_tile_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_s2g_tile_override_addr_param_4];
; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_tile_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_s2g_tile_override_addr_param_5];
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_tile_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_s2g_tile_override_addr_param_6];
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_tile_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_s2g_tile_override_addr_param_7];
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_override_addr_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_override_addr_param_8];
; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
ret void
}
-define void @cp_async_bulk_tensor_s2g_im2col_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_s2g_im2col_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_s2g_im2col_override_addr(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_2];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_5];
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_6];
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_7];
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_s2g_im2col_override_addr_param_8];
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
; CHECK-NEXT: ret;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_im2col_override_addr(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
ret void
}
-define void @cp_async_bulk_tensor_s2g_im2col_w_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_s2g_im2col_w_override_addr(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_s2g_im2col_w_override_addr(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_2];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_5];
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_6];
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_7];
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_s2g_im2col_w_override_addr_param_8];
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs::w.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_im2col_w_override_addr(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
ret void
}
-define void @cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+define void @cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
; CHECK-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<7>;
@@ -157,62 +155,61 @@ define void @cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(ptr addrspac
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_1];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_2];
-; CHECK-NEXT: ld.param.b16 %rs1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_3];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_13];
; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim [%rd2, %rd3, {%rs1}, {%r1}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_4];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_8];
-; CHECK-NEXT: ld.param.b16 %rs3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_12];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_14];
; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_5];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_9];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: ld.param::func.b16 %rs4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_15];
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs5, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_6];
-; CHECK-NEXT: ld.param.b32 %r6, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_10];
-; CHECK-NEXT: ld.param.b32 %r7, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: ld.param::func.b16 %rs5, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r6, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param::func.b32 %r7, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_16];
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1];
-; CHECK-NEXT: ld.param.b16 %rs6, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_7];
-; CHECK-NEXT: ld.param.b32 %r8, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_11];
-; CHECK-NEXT: ld.param.b32 %r9, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_17];
+; CHECK-NEXT: ld.param::func.b16 %rs6, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r8, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param::func.b32 %r9, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_17];
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_18];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride_param_18];
; CHECK-NEXT: cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim.L2::cache_hint [%rd2, %rd3, {%rs1}, {%r1}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], [%rd1], %rd4;
; CHECK-NEXT: cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group.override::global_address.override::global_dim_stride.L2::cache_hint [%rd2, %rd3, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_tile_override_addr_dim_stride(
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
-; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
; without cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 0, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 0, i1 0)
; with cache hint
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i32 %d0, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
- tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %ogbl_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.1d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.3d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.4d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override.addr.dim.stride.5d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
ret void
}
@@ -223,19 +220,18 @@ define void @cp_async_bulk_tensor_s2g_tile_scatter4_2d(i32 %flag, ptr addrspace(
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_1];
-; CHECK-NEXT: ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_2];
-; CHECK-NEXT: ld.param.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_3];
-; CHECK-NEXT: ld.param.b32 %r1, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_4];
-; CHECK-NEXT: ld.param.b32 %r2, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_5];
-; CHECK-NEXT: ld.param.b32 %r3, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_8];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_8];
; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile::scatter4.bulk_group.override::global_address [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-NEXT: ld.param.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_9];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_s2g_tile_scatter4_2d_param_9];
; CHECK-NEXT: cp.async.bulk.tensor.2d.global.shared::cta.tile::scatter4.bulk_group.override::global_address.L2::cache_hint [%rd2, %rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd4;
; CHECK-NEXT: ret;
-;
; CHECK-FORMAT-LABEL: cp_async_bulk_tensor_s2g_tile_scatter4_2d(
; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 false)
; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(ptr addrspace(3) %src, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
>From 130c078a601f92bf8dce0b7c06eafca8b44f4584 Mon Sep 17 00:00:00 2001
From: Rajat Bajpai <rbajpai at nvidia.com>
Date: Thu, 13 Aug 2026 09:53:12 +0000
Subject: [PATCH 3/3] Fixed formatting nits
---
llvm/include/llvm/IR/IntrinsicsNVVM.td | 2 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 30 ++++++++++--------------
2 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 109b8cd171e51..6e8a289138ed8 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1270,7 +1270,7 @@ class NVVMPureIntrinsic<list<LLVMType> ret_types,
// For im2col mode the number of offsets is (dim - 2).
// For im2col_w / im2col_w_128 modes it is always 2 (wHalo and wOffset),
-// irrespective of dim.
+// irrespective of dim.
// For all other modes (e.g. tile) there are no offsets.
class TMA_IM2COL_OFFSETS_UTIL<int dim, string mode> {
int num = !cond(
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 338cdd2b4d13b..2f837475101bf 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -655,7 +655,7 @@ class TMA_IM2COL_UTIL<int dim, string mode> {
// im2col -> im2col_no_offs
// im2col_w -> im2col_no_offs::w
// tile_scatter4 -> tile::scatter4
-class TMA_MODE_ASM_UTIL<string mode> {
+class TMA_S2G_MODE_ASM_UTIL<string mode> {
string ret = !cond(
!eq(mode, "im2col") : "im2col_no_offs",
!eq(mode, "im2col_w") : "im2col_no_offs::w",
@@ -828,7 +828,7 @@ multiclass TMA_TENSOR_S2G_INTR<int dim, string mode,
(intr B64:$ch, -1));
// Fix-up the asm_str when it is im2col/scatter4.
- defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
+ defvar mode_asm_str = TMA_S2G_MODE_ASM_UTIL<mode>.ret;
defvar prefix = "cp.async.bulk.tensor"
# "." # dim_val # "d"
# ".global.shared::cta"
@@ -911,7 +911,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode,
string override> {
defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
defvar is_scatter4 = !eq(mode, "tile_scatter4");
- defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
+ defvar mode_asm_str = TMA_S2G_MODE_ASM_UTIL<mode>.ret;
defvar inst_name = "cp.async.bulk.tensor." # !if(is_scatter4, 2, dim) #
"d.global.shared::cta." # mode_asm_str # ".bulk_group";
defvar suffix = base.override_suffix;
@@ -943,7 +943,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode,
defvar intr_dag_ch =
!con(intr_dag_base, (intr i64:$ch, -1));
- let Predicates = [callSubtarget<"hasRubinFamilySupport">] in {
+ let Predicates = [hasRubinFamilySupport] in {
def "" : NVPTXInst<(outs), ins_dag,
inst_name # suffix # asm_operands # ";",
[intr_dag]>;
@@ -964,7 +964,7 @@ def tma_tensor_reduction_imm :
multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
string override> {
defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
- defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
+ defvar mode_asm_str = TMA_S2G_MODE_ASM_UTIL<mode>.ret;
defvar inst_name = "cp.reduce.async.bulk.tensor." # dim #
"d.global.shared::cta";
defvar suffix = "." # mode_asm_str # ".bulk_group" # base.override_suffix;
@@ -1000,7 +1000,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
!con(intr_dag_base,
(intr B64:$ch, tma_tensor_reduction_imm:$red_op, -1));
- let Predicates = [callSubtarget<"hasRubinFamilySupport">] in {
+ let Predicates = [hasRubinFamilySupport] in {
def "" : NVPTXInst<(outs), ins_dag,
inst_name # "${red_op}" # suffix # asm_operands # ";",
[intr_dag]>;
@@ -1017,21 +1017,17 @@ foreach dim = 1...5 in {
foreach override = !if(!eq(mode, "tile"),
["override_addr", "override_addr_dim_stride"],
["override_addr"]) in {
- defm S2G_ # dim # "D_" #
- !toupper(!subst("::", "_", mode) # "_" # override)
- : NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<
- dim, mode, override>;
- defm RED_ # dim # "D_" #
- !toupper(!subst("::", "_", mode) # "_" # override)
- : NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<
- dim, mode, override>;
+ defvar suffix = dim # "D_" # !toupper(mode # "_" # override);
+ defm S2G_ # suffix
+ : NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<dim, mode, override>;
+ defm RED_ # suffix
+ : NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<dim, mode, override>;
}
}
}
defm S2G_2D_TILE_SCATTER4_OVERRIDE_ADDR
- : NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<
- 5, "tile_scatter4", "override_addr">;
+ : NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<5, "tile_scatter4", "override_addr">;
// TMA Copy from Shared to Global memory with Reduction
multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, string mode,
@@ -1040,7 +1036,7 @@ multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, string mode,
defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
- defvar mode_asm_str = TMA_MODE_ASM_UTIL<mode>.ret;
+ defvar mode_asm_str = TMA_S2G_MODE_ASM_UTIL<mode>.ret;
defvar prefix = "cp.reduce.async.bulk.tensor"
# "." # dim # "d"
# ".global.shared::cta";
More information about the llvm-commits
mailing list