[llvm] [LLVM][NVPTX] Add support for TMA prefetch Rubin extensions (PR #217630)
Rajat Bajpai via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 03:13:01 PDT 2026
https://github.com/rajatbajpai updated https://github.com/llvm/llvm-project/pull/217630
>From c4da187ca244c17c31039e98dff90432f8bcd6c6 Mon Sep 17 00:00:00 2001
From: Rajat Bajpai <rbajpai at nvidia.com>
Date: Tue, 18 Aug 2026 06:58:03 +0000
Subject: [PATCH 1/2] [LLVM][NVPTX] Add support for TMA prefetch Rubin
extensions
This change adds support for the following operations:
1. Bulk prefetch intrinsic with eviction-priority policies
(evict_normal and evict_last).
2. Tensor prefetch intrinsics with eviction-priority policies,
including applicable override variants.
3. Tensor prefetch override intrinsics with optional cache hints.
4. Bulk applypriority intrinsic with the evict_normal policy.
5. Tensor applypriority intrinsics with the evict_normal policy,
including applicable override variants.
---
llvm/docs/NVPTXUsage.md | 409 ++++++++++++++++--
llvm/include/llvm/IR/IntrinsicsNVVM.td | 209 ++++++++-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 18 +
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 10 +
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp | 12 +
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.h | 2 +
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 261 +++++++++--
...iority-async-bulk-global-evict-priority.ll | 45 ++
...iority-async-bulk-tensor-evict-priority.ll | 252 +++++++++++
...ync-bulk-tensor-override-evict-priority.ll | 225 ++++++++++
.../cp-async-bulk-prefetch-evict-priority.ll | 53 +++
...ync-bulk-tensor-prefetch-evict-priority.ll | 328 ++++++++++++++
...tensor-prefetch-override-evict-priority.ll | 245 +++++++++++
.../cp-async-bulk-tensor-prefetch-override.ll | 324 ++++++++++++++
14 files changed, 2337 insertions(+), 56 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/applypriority-async-bulk-global-evict-priority.ll
create mode 100644 llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-evict-priority.ll
create mode 100644 llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-override-evict-priority.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cp-async-bulk-prefetch-evict-priority.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-evict-priority.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override-evict-priority.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override.ll
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index efad04b638494..9802fb6af638d 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1621,6 +1621,38 @@ of the permutation is defined as follows:
### TMA family of Intrinsics
+#### Tensor-Map Property Overrides
+
+Tensor intrinsics with `override` in their name replace properties in the
+opaque tensor map with explicit operands. The supported override variants and
+operand layouts are:
+
+| Intrinsic suffix | Modes and dimensions | Operands after `%tensor_map` |
+|------------------|----------------------|------------------------------|
+| `override.addr` | `tile` 1D--5D; `im2col`, `im2col.w`, and `im2col.w.128` 3D--5D; `tile.gather4` or `tile.scatter4` 2D | `ptr addrspace(1) %override_addr` |
+| `override.addr.dim` | `tile` 1D | `%override_addr, i16 %ts0` |
+| `override.addr.dim.stride` | `tile` 2D--5D | `%override_addr, i16 %ts0 ... i16 %ts{N-1}, i32 %stride0 ... i32 %stride{N-2}, i16 %upper_stride` |
+
+`%override_addr` replaces the tensor map's global base address. It must be
+16-byte aligned; otherwise a runtime error is raised. The 128 KiB range
+beginning at that address must be allocated and accessible while the
+instruction executes; otherwise the behavior is undefined.
+
+`%ts0 ... %ts{N-1}` replace the global dimensions. For
+`override.addr.dim.stride`, `%stride0 ... %stride{N-2}` contain bits 4--35 of
+the corresponding byte strides and `%upper_stride` contains bits 36--51 of
+every stride. Equivalently, stride `i` is
+`((zext(%upper_stride) << 32) | zext(%stride{i})) << 4` bytes. Tensor
+coordinates must be zero when dimensions or strides are overridden; otherwise
+the behavior is undefined.
+
+These variants lower to `.override::global_address`, optionally followed by
+`.override::global_dim` or `.override::global_dim_stride`, and require
+`sm_107f` and PTX 9.4 or later.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-overriding-tensor-property-value).
+
#### '`llvm.nvvm.cp.async.bulk.global.to.shared.cluster`'
##### Syntax:
@@ -1739,6 +1771,222 @@ prefetched in terms of bytes and it must be a multiple of 16.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch).
+#### '`llvm.nvvm.cp.async.bulk.prefetch.evict.priority`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 %size, i32 %evict_policy)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`' intrinsic
+asynchronously prefetches `%size` bytes from global memory into the L2 cache
+using the selected eviction priority. It lowers to the
+`cp.async.bulk.prefetch.L2.global` PTX instruction with an L2 eviction-priority
+qualifier. Unlike `@llvm.nvvm.cp.async.bulk.prefetch.L2`, it takes an eviction
+policy instead of a cache hint. `%size` must be a multiple of 16.
+
+The trailing `i32 %evict_policy` is an immediate (`immarg`) argument that
+selects the L2 eviction policy.
+
+| Value | Name | PTX qualifier |
+|-------|----------------|---------------------|
+| 0 | `evict_normal` | (none, the default) |
+| 1 | `evict_last` | `.L2::evict_last` |
+
+This intrinsic requires `sm_107f` and PTX 9.4 or later.
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.1d(ptr %tensor_map, i32 %d0, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.2d(ptr %tensor_map, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 %evict_policy)
+```
+
+##### Overview:
+
+These intrinsics asynchronously prefetch tile-mode tensor data from global
+memory into the L2 cache using the selected eviction priority. They are the
+eviction-priority forms of the corresponding
+`@llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.*` intrinsics, and the trailing
+`i32 %evict_policy` uses the encoding described for
+`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`. They do not take cache-hint
+operands and require `sm_107f` and PTX 9.4 or later.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.3d(ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.3d(ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.3d(ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+These intrinsics asynchronously prefetch im2col-mode tensor data from global
+memory into the L2 cache using the selected eviction priority. An N-dimensional
+form takes N `i32` tensor coordinates. The `im2col` forms take N - 2 `i16`
+im2col offsets, while the `im2col.w` and `im2col.w.128` forms take
+`i16 %wHalo` and `i16 %wOffset`.
+
+The trailing `i32 %evict_policy` uses the encoding described for
+`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`. These forms do not take
+cache-hint operands and require `sm_107f` and PTX 9.4 or later.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override*.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i64 %cache_hint, i1 %flag_cache_hint)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %cache_hint, i1 %flag_cache_hint)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.1d(ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %cache_hint, i1 %flag_cache_hint)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.2d(ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i64 %cache_hint, i1 %flag_cache_hint)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.3d(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.4d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+These intrinsics asynchronously prefetch tile-mode tensor data from global
+memory into the L2 cache while overriding tensor-map properties with explicit
+operands. They lower to the
+`cp.async.bulk.prefetch.tensor.<N>d.L2.global.tile` PTX instructions qualified
+with `.override::*`.
+
+The supported override variants and operands are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
+
+- The last argument is a compile-time boolean indicating whether `%cache_hint`
+ is valid. When set, it
+ indicates a valid cache_hint (`i64 %ch`) and generates the
+ `.L2::cache_hint` variant of the PTX instruction.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.3d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %cache_hint, i1 %flag_cache_hint)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.4d(..., i16 %im2col0, i16 %im2col1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(..., i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+```
+
+##### Overview:
+
+These intrinsics asynchronously prefetch im2col-mode tensor data from global
+memory into the L2 cache while overriding the tensor-map global address. An
+N-dimensional form takes N `i32` tensor coordinates. The `im2col` forms take
+N - 2 `i16` im2col offsets, while the `im2col.w` and `im2col.w.128` forms take
+`i16 %wHalo` and `i16 %wOffset`.
+
+`%override_addr` and its requirements are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides). The trailing
+`i1 %flag_cache_hint` must be a compile-time constant; when set, it marks
+`i64 %cache_hint` as valid and emits the `.L2::cache_hint` PTX qualifier.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override*.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.1d(ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.2d(ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.3d(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.4d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.5d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 %evict_policy)
+```
+
+##### Overview:
+
+These intrinsics asynchronously prefetch tile-mode tensor data from global
+memory into the L2 cache using the selected eviction priority while overriding
+tensor-map properties. They are the eviction-priority forms of the
+corresponding tensor prefetch override intrinsics. The override operands are
+described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides). The remaining
+operands use the coordinate layout of the corresponding tile or
+`tile.gather4` intrinsic.
+
+The trailing `i32 %evict_policy` is an immediate argument: 0 selects the
+default `evict_normal` policy and emits no PTX qualifier; 1 selects
+`.L2::evict_last`. These forms do not take cache-hint operands.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.3d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i32 %evict_policy)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.4d(..., i16 %im2col0, i16 %im2col1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.5d(..., i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+```
+
+##### Overview:
+
+These intrinsics asynchronously prefetch im2col-mode tensor data from global
+memory into the L2 cache using the selected eviction priority while overriding
+the tensor-map global address. An N-dimensional form takes N `i32` tensor
+coordinates. The `im2col` forms take N - 2 `i16` im2col offsets, while the
+`im2col.w` and `im2col.w.128` forms take `i16 %wHalo` and `i16 %wOffset`.
+
+`%override_addr` and its requirements are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides). The trailing
+`i32 %evict_policy` uses the encoding described for
+`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`; these forms do not take
+cache-hint operands.
+
+For more information, refer to the
+[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+
#### '`llvm.nvvm.prefetch.*`'
##### Syntax:
@@ -1803,8 +2051,129 @@ operand size is an integer constant that specifies the amount of data, in bytes,
in the specified cache level on which the priority is to be applied. The only
supported value for the size operand is 128.
+#### '`llvm.nvvm.applypriority.async.bulk.evict.priority`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1) %global_ptr, i32 %size, i32 %evict_policy)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.applypriority.async.bulk.evict.priority`' intrinsic applies
+the selected L2 eviction priority to
+\[%global_ptr, %global_ptr + %size). It lowers to the
+`applypriority.async.bulk.global.bulk_group` PTX instruction. `%size` may be a
+register or an immediate and must be a multiple of 16. `%global_ptr` must be
+aligned to 128 bytes.
+
+`%evict_policy` is an immediate argument. Its only valid value is 0, which
+selects `.L2::evict_normal`.
+
+This intrinsic requires `sm_107f` and PTX 9.4 or later.
+
+#### '`llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.1d(ptr %tensor_map, i32 %d0, i32 %evict_policy)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.2d(ptr %tensor_map, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 %evict_policy)
+```
+
+##### Overview:
+
+These intrinsics apply `evict_normal` priority to a tile tensor region. An
+N-dimensional tile form takes N `i32` tensor coordinates; the `tile.gather4`
+form takes `i32 %x0` followed by four row coordinates `%y0 ... %y3`. The
+trailing `%evict_policy` is an immediate argument whose only valid value is 0.
+They require `sm_107f` and PTX 9.4 or later.
+
+#### '`llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.3d(ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i32 %evict_policy)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.4d(..., i16 %im2col0, i16 %im2col1, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.5d(..., i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+
+```
+
+##### Overview:
+
+These intrinsics apply `evict_normal` priority to an im2col tensor region
+described by `%tensor_map`. An N-dimensional form takes N `i32` tensor
+coordinates. The `im2col` forms take N - 2 `i16` im2col offsets, while the
+`im2col.w` and `im2col.w.128` forms take `i16 %wHalo` and `i16 %wOffset`.
+
+The trailing `%evict_policy` is an immediate argument whose only valid value is
+0. These intrinsics require `sm_107f` and PTX 9.4 or later.
+
+#### '`llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override*.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.1d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %evict_policy)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.1d(ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i32 %evict_policy)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.2d(ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i32 %evict_policy)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.3d(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.4d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.5d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.override.addr.2d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 %evict_policy)
+```
+
+##### Overview:
+
+These intrinsics apply the selected L2 eviction priority to a tile region while
+overriding tensor-map properties with explicit operands. Their supported
+override variants and operands are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides). An
+N-dimensional tile form takes N `i32` tensor coordinates; the `tile.gather4`
+form takes `i32 %x0` followed by four row coordinates `%y0 ... %y3`.
+
+`%evict_policy` is an immediate argument whose only valid value is 0
+(`evict_normal`).
+
For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-applypriority).
+#### '`llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.3d(ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i32 %evict_policy)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.4d(..., i16 %im2col0, i16 %im2col1, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.5d(..., i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
+```
+
+##### Overview:
+
+These intrinsics apply the selected L2 eviction priority to an im2col region
+while overriding the tensor-map global address. An N-dimensional form takes N
+`i32` tensor coordinates. The `im2col` forms take N - 2 `i16` im2col offsets,
+while the `im2col.w` and `im2col.w.128` forms take `i16 %wHalo` and
+`i16 %wOffset`.
+
+`%override_addr` and its requirements are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
+`%evict_policy` is an immediate argument whose only valid value is 0
+(`evict_normal`).
+
#### `llvm.nvvm.discard.*`'
##### Syntax:
@@ -2284,23 +2653,9 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(
The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override*.[1-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.[1-5]d.*` set of PTX instructions qualified with the `.override::*` qualifiers.
These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `tile` mode, while overriding specific properties of the opaque tensor-map object with explicit instruction operands.
-The `.override::*` qualifiers allow certain tensor-map properties to be ignored and instead supplied as explicit operands.
-The supported override variants are:
-
-- `override.addr` (`.override::global_address`): The global base address from the tensor-map is ignored and the explicit `ptr addrspace(1) %override_addr` operand is used instead.
- The address must be 16B aligned; otherwise a runtime error is raised.
- The memory range `[%override_addr, %override_addr + 128 KiB)` must be allocated and accessible during execution; otherwise the behavior is undefined.
-
-- `override.addr.dim` (`.override::global_address` + `.override::global_dim`, 1D only): Overrides the base address and the tensor global dimension.
- The `i16 %ts0` operand specifies the new tensor size.
- This variant requires the `tile` load mode; the base address override is mandatory when overriding attributes.
- The tensor start coordinates must be zero; otherwise the behavior is undefined.
-
-- `override.addr.dim.stride` (`.override::global_address` + `.override::global_dim_stride`, 2D and higher): Overrides the base address, the tensor global dimensions, and the global strides.
- The `i16 %ts0 ... i16 %ts{dim-1}` operands specify the new tensor sizes, the `i32 %stride0 ... i32 %stride{dim-2}` operands specify the lower strides, and the `i16 %upper_stride` operand specifies the upper stride.
- The effective global stride is computed as `global_stride[i] = ((%stride{i} + (%upper_stride{i} << 32)) << 4)`.
- This variant requires the `tile` load mode; the base address override is mandatory when overriding attributes.
- The tensor start coordinates must be zero; otherwise the behavior is undefined.
+The `.override::*` qualifiers ignore certain tensor-map properties and instead take them as explicit instruction operands.
+The supported override variants, operands, and restrictions are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
The `scatter4.override.addr` variant combines the `tile.scatter4` mode (2D only) with the base address override.
In `tile.scatter4` mode, a single 2D source tensor is divided into four rows in the 2D destination tensor.
@@ -2326,9 +2681,9 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(..., i3
##### Overview:
The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.[3-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.im2col_no_offs.[3-5]d.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
-These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `im2col` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
-The address must be 16B aligned and the memory range `[%override_addr, %override_addr + 128 KiB)` must be allocated and accessible during execution; otherwise the behavior is undefined.
-The override address semantics and the last boolean flag argument have the same functionality as described in the `s2g.tile` override intrinsics above.
+They copy tensor data from shared::cta to global memory in `im2col` mode using
+the `override.addr` operands described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
+The last boolean flag argument has the same functionality as described in the `s2g.tile` override intrinsics above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
@@ -2345,8 +2700,9 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(...,
##### Overview:
The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.[3-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.im2col_no_offs::w.[3-5]d.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
-These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `im2col_w` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
-The override address semantics and the last boolean flag argument have the same functionality as described in the `s2g.tile` override intrinsics above.
+They copy tensor data from shared::cta to global memory in `im2col_w` mode using
+the `override.addr` operands described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
+The last boolean flag argument has the same functionality as described in the `s2g.tile` override intrinsics above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
@@ -2376,7 +2732,8 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override.addr.dim.strid
The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.tile.override*.[1-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.[1-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::*` qualifiers.
These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `tile` mode, while overriding specific properties of the opaque tensor-map object with explicit instruction operands.
-The three override variants (`override.addr`, `override.addr.dim`, and `override.addr.dim.stride`) have the same semantics as described in the `s2g.tile` override intrinsics above.
+The supported override variants and operands are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
The `i32 %red_op` argument selects the reduction operation to perform.
It must be a compile-time constant in the half-open range `[0, 8)`, with the following encoding:
@@ -2414,7 +2771,8 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(...,
The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.[3-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.im2col_no_offs.[3-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `im2col` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
-The `i32 %red_op` argument, the override address semantics, and the last boolean flag argument have the same functionality as described in the `reduce.tile` override intrinsics above.
+The `override.addr` operands are described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
+`%red_op` and the last boolean flag are as described for `reduce.tile` above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
@@ -2432,7 +2790,8 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(..
The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.[3-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.im2col_no_offs::w.[3-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `im2col_w` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
-The `i32 %red_op` argument, the override address semantics, and the last boolean flag argument have the same functionality as described in the `reduce.tile` override intrinsics above.
+The `override.addr` operands are described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
+`%red_op` and the last boolean flag are as described for `reduce.tile` above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 69905899f3007..f9db6a8455230 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1355,6 +1355,12 @@ class CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<int dim, string mode, string override>
string Suffix = "_" # mode # "_" # override # "_" # dim # "d";
}
+// Helper that emits the intrinsic-property set for the cache_hint flag argument.
+class NVVM_CACHE_HINT_ARGPROP<int argNo> {
+ IntrinsicProperty Prop =
+ ArgInfo<ArgIndex<argNo>, [ArgName<"flag_cache_hint">]>;
+}
+
class CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode, string override>
: CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<dim, mode, override> {
string Name = Prefix # "s2g" # Suffix;
@@ -1371,7 +1377,7 @@ class CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode, string overri
list<IntrinsicProperty> IntrProp = [
IntrConvergent,
ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>, WriteOnly<ArgIndex<2>>,
- ArgInfo<ArgIndex<FlagCacheHintIdx>, [ArgName<"flag_cache_hint">]>
+ NVVM_CACHE_HINT_ARGPROP<FlagCacheHintIdx>.Prop
];
}
@@ -1389,10 +1395,85 @@ class CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode, string ove
Range<ArgIndex<RedOpIdx>, 0, 8>,
ArgInfo<ArgIndex<RedOpIdx>, [ArgName<"red_op">,
ImmArgPrinter<"printTMAReductionOp">]>,
- ArgInfo<ArgIndex<!add(RedOpIdx, 1)>, [ArgName<"flag_cache_hint">]>
+ NVVM_CACHE_HINT_ARGPROP<!add(RedOpIdx, 1)>.Prop
+ ];
+}
+
+// Helper that emits the intrinsic-property set for the evict_policy argument.
+// `upperBound` is the exclusive upper bound of the allowed policy value range.
+// Prefetch accepts both EVICT_NORMAL and EVICT_LAST (default 2).
+// Applypriority only accepts EVICT_NORMAL (upperBound = 1).
+class NVVM_EVICT_POLICY_ARGPROP<int argNo, int upperBound = 2> {
+ list<IntrinsicProperty> Prop = [
+ ImmArg<ArgIndex<argNo>>,
+ Range<ArgIndex<argNo>, 0, upperBound>,
+ ArgInfo<ArgIndex<argNo>, [ArgName<"evict_policy">,
+ ImmArgPrinter<"printEvictPolicyType">]>
];
}
+class CP_ASYNC_BULK_TENSOR_PREFETCH_OVERRIDE_INTR<int dim, string mode,
+ string override,
+ bit IsEvict = 0>
+ : CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<dim, mode, override> {
+ string Name = Prefix # "prefetch" # !if(IsEvict, "_evict_priority", "") # Suffix;
+
+ list<LLVMType> ArgsTy = !listconcat(
+ [llvm_ptr_ty], // tensormap_ptr
+ OverrideTypes, // override_addr (src)
+ TensorDimsTy, // actual tensor dims
+ Im2ColOffsetsTy, // im2col offsets
+ !if(IsEvict, [llvm_i32_ty], [llvm_i64_ty]));
+
+ list<LLVMType> FlagsTy = !if(IsEvict, [], [llvm_i1_ty]);
+
+ defvar PolicyIdx = !sub(!size(ArgsTy), 1);
+ list<IntrinsicProperty> IntrProp = !listconcat(
+ [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>],
+ !if(IsEvict,
+ NVVM_EVICT_POLICY_ARGPROP<PolicyIdx>.Prop,
+ [NVVM_CACHE_HINT_ARGPROP<!size(ArgsTy)>.Prop]));
+}
+
+// Applypriority TMA variants.
+class APPLYPRIORITY_BULK_TENSOR_INTR<int dim, string mode> {
+ string Name = "int_nvvm_applypriority_async_bulk_tensor_evict_priority_" #
+ mode # "_" # !if(!eq(mode, "tile_gather4"), 2, dim) # "d";
+
+ int NumIm2ColOffsets = TMA_IM2COL_OFFSETS_UTIL<dim, mode>.num;
+ list<LLVMType> ArgsTy = !listconcat(
+ [llvm_ptr_ty], // tensormap_ptr
+ !listsplat(llvm_i32_ty, dim), // actual tensor dims
+ !listsplat(llvm_i16_ty, NumIm2ColOffsets), // im2col offsets
+ [llvm_i32_ty]); // evict_policy
+
+ defvar PolicyIdx = !sub(!size(ArgsTy), 1);
+ list<IntrinsicProperty> IntrProp = !listconcat(
+ [IntrConvergent, ReadOnly<ArgIndex<0>>],
+ NVVM_EVICT_POLICY_ARGPROP<PolicyIdx, 1>.Prop);
+}
+
+// Applypriority TMA variants with override global address.
+class APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<int dim, string mode,
+ string override>
+ : CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<dim, mode, override> {
+ string Name = "int_nvvm_applypriority_async_bulk_tensor_evict_priority_" #
+ mode # "_" # override # "_" #
+ !if(!eq(mode, "tile_gather4"), 2, dim) # "d";
+
+ list<LLVMType> ArgsTy = !listconcat(
+ [llvm_ptr_ty], // tensormap_ptr
+ OverrideTypes, // override_addr (src)
+ TensorDimsTy, // actual tensor dims
+ Im2ColOffsetsTy, // im2col offsets
+ [llvm_i32_ty]); // evict_policy
+
+ defvar PolicyIdx = !sub(!size(ArgsTy), 1);
+ list<IntrinsicProperty> IntrProp = !listconcat(
+ [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>],
+ NVVM_EVICT_POLICY_ARGPROP<PolicyIdx, 1>.Prop);
+}
+
let TargetPrefix = "nvvm" in {
//
@@ -3033,8 +3114,7 @@ def int_nvvm_cp_async_bulk_tensor_s2g_tile_scatter4_override_addr_2d
[llvm_i64_ty]), // cache_hint
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
- WriteOnly<ArgIndex<2>>, ArgInfo<ArgIndex<9>,
- [ArgName<"flag_cache_hint">]>]>;
+ WriteOnly<ArgIndex<2>>, NVVM_CACHE_HINT_ARGPROP<9>.Prop]>;
// TMA Tensor Copy Intrinsics: G2S -> From Global to Shared memory variants
foreach dim = 1...5 in {
@@ -3077,6 +3157,34 @@ foreach dim = 1...5 in {
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>]>;
+ defvar evict_policy_idx = !add(!add(1, !size(tensor_dim_args)),
+ !size(im2col_offsets_args));
+ def int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_ # mode # _ # dim # d :
+ DefaultAttrsIntrinsic<[],
+ !listconcat([llvm_ptr_ty], // tensormap_ptr
+ tensor_dim_args, // actual tensor dims
+ im2col_offsets_args, // im2col offsets
+ [llvm_i32_ty]), // evict_policy
+ !listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>],
+ NVVM_EVICT_POLICY_ARGPROP<evict_policy_idx>.Prop)>;
+
+ // Prefetch + Override
+ foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
+ foreach prefetch_override =
+ [CP_ASYNC_BULK_TENSOR_PREFETCH_OVERRIDE_INTR<dim, mode, override>] in
+ def prefetch_override.Name
+ : DefaultAttrsIntrinsicFlags<[], prefetch_override.ArgsTy,
+ prefetch_override.FlagsTy,
+ prefetch_override.IntrProp>;
+ // Prefetch + Evict priority + Override
+ foreach prefetch_evict_override =
+ [CP_ASYNC_BULK_TENSOR_PREFETCH_OVERRIDE_INTR<dim, mode, override, /*IsEvict=*/1>] in
+ def prefetch_evict_override.Name
+ : DefaultAttrsIntrinsicFlags<[], prefetch_evict_override.ArgsTy,
+ prefetch_evict_override.FlagsTy,
+ prefetch_evict_override.IntrProp>;
+ }
+
def int_nvvm_cp_async_bulk_tensor_g2s_cta_ # mode # _ # dim # d :
DefaultAttrsIntrinsicFlags<[],
!listconcat([llvm_shared_ptr_ty, // dst_ptr
@@ -3129,6 +3237,36 @@ def int_nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_2d
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>]>;
+// TMA prefetch for tile::gather4 with eviction priority
+def int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_tile_gather4_2d
+ : DefaultAttrsIntrinsic<[],
+ !listconcat([llvm_ptr_ty], // tensormap_ptr
+ !listsplat(llvm_i32_ty, 5), // co-ordinates
+ [llvm_i32_ty]), // evict_policy
+ !listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>],
+ NVVM_EVICT_POLICY_ARGPROP<6>.Prop)>;
+
+// TMA prefetch for tile::gather4 with override address
+def int_nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_override_addr_2d
+ : DefaultAttrsIntrinsicFlags<[],
+ !listconcat([llvm_ptr_ty, // tensormap_ptr
+ llvm_global_ptr_ty], // override_addr (src)
+ !listsplat(llvm_i32_ty, 5), // co-ordinates
+ [llvm_i64_ty]), // cache_hint
+ [llvm_i1_ty], // Flag for cache_hint
+ [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
+ NVVM_CACHE_HINT_ARGPROP<8>.Prop]>;
+
+// TMA prefetch for tile::gather4 with override address and eviction priority
+def int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_tile_gather4_override_addr_2d
+ : DefaultAttrsIntrinsic<[],
+ !listconcat([llvm_ptr_ty, // tensormap_ptr
+ llvm_global_ptr_ty], // override_addr (src)
+ !listsplat(llvm_i32_ty, 5), // co-ordinates
+ [llvm_i32_ty]), // evict_policy
+ !listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>],
+ NVVM_EVICT_POLICY_ARGPROP<7>.Prop)>;
+
// Intrinsics for Prefetch and Prefetchu
let IntrProperties = [IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>] in {
foreach level = ["L1", "L2"] in {
@@ -3155,6 +3293,59 @@ let IntrProperties = [IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<
: DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty]>;
}
+// Aapplypriority bulk variants.
+def int_nvvm_applypriority_async_bulk_evict_priority
+ : DefaultAttrsIntrinsic<[],
+ [llvm_global_ptr_ty, // global_ptr
+ llvm_i32_ty, // size (byte length; must be a multiple of 16)
+ llvm_i32_ty], // evict_policy
+ !listconcat([IntrConvergent, IntrArgMemOnly,
+ ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
+ NVVM_EVICT_POLICY_ARGPROP<2, 1>.Prop)>;
+
+// Applypriority TMA variants.
+foreach dim = 1...5 in {
+ foreach mode = !if(!ge(dim, 3),
+ ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach ap_tensor =
+ [APPLYPRIORITY_BULK_TENSOR_INTR<dim, mode>] in
+ def ap_tensor.Name : DefaultAttrsIntrinsic<[], ap_tensor.ArgsTy,
+ ap_tensor.IntrProp>;
+ }
+}
+// Applypriority TMA variants for tile::gather4.
+def int_nvvm_applypriority_async_bulk_tensor_evict_priority_tile_gather4_2d
+ : DefaultAttrsIntrinsic<[],
+ !listconcat([llvm_ptr_ty], // tensormap_ptr
+ !listsplat(llvm_i32_ty, 5), // co-ordinates (d0..d4)
+ [llvm_i32_ty]), // evict_policy
+ !listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>],
+ NVVM_EVICT_POLICY_ARGPROP<6, 1>.Prop)>;
+
+// Applypriority TMA variants with override global address.
+foreach dim = 1...5 in {
+ foreach mode = !if(!ge(dim, 3),
+ ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
+ foreach ap_tensor_override =
+ [APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<dim, mode, override>] in
+ def ap_tensor_override.Name
+ : DefaultAttrsIntrinsic<[], ap_tensor_override.ArgsTy,
+ ap_tensor_override.IntrProp>;
+ }
+ }
+}
+// Applypriority TMA variants for tile::gather4 with override global address.
+def int_nvvm_applypriority_async_bulk_tensor_evict_priority_tile_gather4_override_addr_2d
+ : DefaultAttrsIntrinsic<[],
+ !listconcat([llvm_ptr_ty], // tensormap_ptr
+ [llvm_global_ptr_ty], // override_addr (src)
+ !listsplat(llvm_i32_ty, 5), // co-ordinates (d0..d4)
+ [llvm_i32_ty]), // evict_policy
+ !listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>,
+ ReadOnly<ArgIndex<1>>],
+ NVVM_EVICT_POLICY_ARGPROP<7, 1>.Prop)>;
+
// discard
let IntrProperties = [NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<1>>, IntrHasSideEffects] in {
def int_nvvm_discard_global_L2 : DefaultAttrsIntrinsic<[], [llvm_global_ptr_ty, llvm_i64_ty]>;
@@ -3232,6 +3423,16 @@ def int_nvvm_cp_async_bulk_prefetch_L2
[IntrConvergent, IntrArgMemOnly,
NoCapture<ArgIndex<0>>, ReadOnly<ArgIndex<0>>]>;
+// Intrinsics for Bulk Copy Prefetch with Eviction Priority
+def int_nvvm_cp_async_bulk_prefetch_evict_priority
+ : DefaultAttrsIntrinsic<[],
+ [llvm_global_ptr_ty, // src_gmem_ptr
+ llvm_i32_ty, // copy_size
+ llvm_i32_ty], // evict_policy
+ !listconcat([IntrConvergent, IntrArgMemOnly,
+ NoCapture<ArgIndex<0>>, ReadOnly<ArgIndex<0>>],
+ NVVM_EVICT_POLICY_ARGPROP<2>.Prop)>;
+
def int_nvvm_griddepcontrol_launch_dependents : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
def int_nvvm_griddepcontrol_wait : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index b38ebc3e2b309..a6f7555be838a 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -73,6 +73,22 @@ enum class CTAGroupKind : uint8_t {
CG_2 = 2, // cta_group::2 modifier
};
+// Eviction priorities applicable for prefetch and applypriority intrinsics.
+enum class EvictPolicyType : uint8_t {
+ EVICT_NORMAL = 0, // default
+ EVICT_LAST = 1,
+};
+
+inline StringRef getEvictPolicyName(EvictPolicyType Policy) {
+ switch (Policy) {
+ case EvictPolicyType::EVICT_NORMAL:
+ return "L2::evict_normal";
+ case EvictPolicyType::EVICT_LAST:
+ return "L2::evict_last";
+ }
+ llvm_unreachable("invalid evict policy");
+}
+
enum class Tcgen05MMAKind : uint8_t {
F16 = 0,
TF32 = 1,
@@ -142,6 +158,8 @@ enum class TensormapFillMode : uint8_t {
LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
+LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal);
+
LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index b5cf462dfc470..b67f70d2568dc 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -16,6 +16,16 @@
using namespace llvm;
using namespace nvvm;
+void nvvm::printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal) {
+ const auto *CI = dyn_cast<ConstantInt>(ImmArgVal);
+ if (!CI ||
+ CI->getZExtValue() > static_cast<uint64_t>(EvictPolicyType::EVICT_LAST)) {
+ OS << "Unsupported evict policy";
+ return;
+ }
+ OS << getEvictPolicyName(static_cast<EvictPolicyType>(CI->getZExtValue()));
+}
+
void nvvm::printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal) {
const auto *CI = dyn_cast<ConstantInt>(ImmArgVal);
if (!CI || CI->getZExtValue() > static_cast<uint64_t>(TMAReductionOp::XOR))
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index fe9e2fa260811..9eb281503916b 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -563,6 +563,18 @@ void NVPTXInstPrinter::printCTAGroup(const MCInst *MI, int OpNum,
llvm_unreachable("Invalid cta_group in printCTAGroup");
}
+void NVPTXInstPrinter::printEvictPolicy(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &, raw_ostream &O,
+ StringRef Modifier) {
+ const auto Policy =
+ static_cast<nvvm::EvictPolicyType>(MI->getOperand(OpNum).getImm());
+ // Evict normal is the default priority policy for prefetch and does not print
+ // a qualifier.
+ if (Policy == nvvm::EvictPolicyType::EVICT_NORMAL)
+ return;
+ O << "." << nvvm::getEvictPolicyName(Policy);
+}
+
void NVPTXInstPrinter::printCallOperand(const MCInst *MI, int OpNum,
const MCSubtargetInfo &, raw_ostream &O,
StringRef Modifier) {
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index 69fff1c1c7402..f6a086e6a3379 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -65,6 +65,8 @@ class NVPTXInstPrinter : public MCInstPrinter {
const MCSubtargetInfo &STI, raw_ostream &O);
void printCTAGroup(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O);
+ void printEvictPolicy(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
+ raw_ostream &O, StringRef Modifier = {});
void printCallOperand(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
void printFTZFlag(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 5e01b251ff9fd..142a0761d41b1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -617,6 +617,25 @@ multiclass CP_ASYNC_BULK_PREFETCH_INTR<bit has_ch> {
defm CP_ASYNC_BULK_PREFETCH : CP_ASYNC_BULK_PREFETCH_INTR<has_ch = 0>;
defm CP_ASYNC_BULK_PREFETCH_CH : CP_ASYNC_BULK_PREFETCH_INTR<has_ch = 1>;
+def PrefetchEvictPolicy : TImmLeaf<i32, [{ return Imm == 0 || Imm == 1; }]>;
+def EvictPolicyOp : Operand<i32> { let PrintMethod = "printEvictPolicy"; }
+
+def CP_ASYNC_BULK_PREFETCH_EVICT_PRIORITY
+ : NVPTXInst<(outs),
+ (ins ADDR:$src, B32:$size, EvictPolicyOp:$policy),
+ "cp.async.bulk.prefetch.L2.global${policy} [$src], $size;",
+ [(int_nvvm_cp_async_bulk_prefetch_evict_priority
+ addr:$src, i32:$size, PrefetchEvictPolicy:$policy)]>,
+ Requires<[hasRubinFamilySupport]>;
+
+def APPLYPRIORITY_ASYNC_BULK_GLOBAL_EVICT_PRIORITY
+ : NVPTXInst<(outs),
+ (ins ADDR:$addr, B32:$size),
+ "applypriority.async.bulk.global.bulk_group.L2::evict_normal [$addr], $size;",
+ [(int_nvvm_applypriority_async_bulk_evict_priority
+ addr:$addr, i32:$size, 0)]>,
+ Requires<[hasRubinFamilySupport]>;
+
//-------------------------------------
// TMA Async Bulk Tensor Copy Functions
//-------------------------------------
@@ -865,7 +884,6 @@ class TMA_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
defvar dims_util = TMA_DIMS_UTIL<dim>;
dag dims_dag = dims_util.ins_dag;
string dims_str = dims_util.base_str;
- bit has_multi_dim = !ge(dim, 2);
// Im2col handling (im2col / im2col_w / im2col_w_128).
defvar im2col_util = TMA_IM2COL_UTIL<dim, mode>;
@@ -874,35 +892,45 @@ class TMA_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
string im2col_str = im2col_util.base_str;
string im2col_component = !if(num_im2col, ", {{" # im2col_str # "}}", "");
- // Tensor size handling
+ // Override-kind recognition. Three kinds are supported:
+ // override_addr -> global address only (no tensor size or strides)
+ // override_addr_dim -> global address + tensor size
+ // override_addr_dim_stride -> global address + tensor size + strides
+ bit is_override_addr = !eq(override, "override_addr");
+ bit is_addr_dim = !eq(override, "override_addr_dim");
bit is_addr_dim_stride = !eq(override, "override_addr_dim_stride");
- dag tensor_size_dag = !if(is_addr_dim_stride,
+ bit has_dim_overrides = !or(is_addr_dim, is_addr_dim_stride);
+
+ assert !or(is_override_addr, is_addr_dim, is_addr_dim_stride),
+ "unknown TMA override kind: " # override;
+
+ // Tensor size handling (emitted for both dimension-override variants).
+ dag tensor_size_dag = !if(has_dim_overrides,
!dag(ins, !listsplat(B16, dim), !foreach(i, !range(dim), "ts" # i)),
(ins));
string tensor_size_str = !interleave(!foreach(i, !range(dim), "$ts" # i), ", ");
- // Stride handling for multi-dimensional tensors
- bit need_strides = !and(is_addr_dim_stride, has_multi_dim);
- dag lower_stride_dag = !if(need_strides,
+ // Stride handling for override_addr_dim_stride.
+ dag lower_stride_dag = !if(is_addr_dim_stride,
!dag(ins, !listsplat(B32, !sub(dim, 1)),
!foreach(i, !range(!sub(dim, 1)), "stride" # i)),
(ins));
string lower_stride_str = !interleave(!foreach(i, !range(!sub(dim, 1)), "$stride" # i), ", ");
- dag upper_stride_dag = !if(need_strides, (ins B16:$upper_stride), (ins));
+ dag upper_stride_dag =
+ !if(is_addr_dim_stride, (ins B16:$upper_stride), (ins));
// Compose assembly string components
- string stride_component = !if(has_multi_dim,
+ string stride_component = !if(is_addr_dim_stride,
"{{" # lower_stride_str # "}}, $upper_stride, ",
"");
- string tensor_size_component = !if(is_addr_dim_stride,
+ string tensor_size_component = !if(has_dim_overrides,
"{{" # tensor_size_str # "}}, " # stride_component,
"");
// Common suffix components
- string dim_suffix = !if(has_multi_dim,
- ".override::global_dim_stride",
- ".override::global_dim");
- string override_suffix = ".override::global_address" # !if(is_addr_dim_stride, dim_suffix, "");
+ string dim_suffix = !if(is_addr_dim_stride, ".override::global_dim_stride",
+ !if(is_addr_dim, ".override::global_dim", ""));
+ string override_suffix = ".override::global_address" # dim_suffix;
}
// TMA Copy from Shared to Global memory (with Override Global Address)
@@ -917,11 +945,8 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode,
defvar asm_operands = " [$tmap, $override_addr, " # base.tensor_size_component #
"{{" # base.dims_str # "}}], [$src]";
- defvar is_override_dim_stride_1d =
- !and(base.is_addr_dim_stride, !eq(dim, 1));
- defvar intr = !cast<Intrinsic>(
- "int_nvvm_cp_async_bulk_tensor_s2g_" # mode # "_" #
- !if(is_override_dim_stride_1d, "override_addr_dim", override) # "_" #
+ defvar intr =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_s2g_" # mode # "_" # override # "_" #
!if(is_scatter4, 2, dim) # "d");
defvar ins_dag_base =
@@ -970,12 +995,8 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
defvar asm_operands = " [$tmap, $override_addr, " # base.tensor_size_component #
"{{" # base.dims_str # "}}], [$src]";
- defvar is_override_dim_stride_1d =
- !and(base.is_addr_dim_stride, !eq(dim, 1));
-
- defvar intr = !cast<Intrinsic>(
- "int_nvvm_cp_async_bulk_tensor_reduce_" # mode # "_" #
- !if(is_override_dim_stride_1d, "override_addr_dim", override) # "_" #
+ defvar intr =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_reduce_" # mode # "_" # override # "_" #
dim # "d");
defvar ins_dag_base =
@@ -1013,9 +1034,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
foreach dim = 1...5 in {
foreach mode = !if(
!ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
- foreach override = !if(!eq(mode, "tile"),
- ["override_addr", "override_addr_dim_stride"],
- ["override_addr"]) in {
+ foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
defvar suffix = dim # "D_" # !toupper(mode # "_" # override);
defm S2G_ # suffix
: NVPTX_CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<dim, mode, override>;
@@ -1129,6 +1148,20 @@ multiclass TMA_TENSOR_PREFETCH_INTR<int dim, string mode,
inst_name # ".L2::cache_hint" # asm_str # ", $ch;",
[intr_dag_with_ch]>,
Requires<pred>;
+
+ defvar intr_evict =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_" #
+ mode # "_" # dim_val # "d");
+ defvar ins_dag_evict = !con((ins B64:$tmap), dims_dag, im2col_dag,
+ (ins EvictPolicyOp:$policy));
+ defvar intr_dag_evict = !con((intr_evict B64:$tmap),
+ !setdagop(dims_dag, intr_evict),
+ !setdagop(im2col_dag, intr_evict),
+ (intr_evict PrefetchEvictPolicy:$policy));
+ def _EV : NVPTXInst<(outs), ins_dag_evict,
+ inst_name # "${policy}" # asm_str # ";",
+ [intr_dag_evict]>,
+ Requires<[hasRubinFamilySupport]>;
}
foreach dim = 1...5 in {
foreach mode = !if(!ge(dim, 3), ["tile", "im2col"], ["tile"]) in {
@@ -1146,6 +1179,180 @@ foreach dim = 3...5 in {
defm TMA_TENSOR_PF_TILE_GATHER4_2D : TMA_TENSOR_PREFETCH_INTR<5, "tile_gather4",
[hasTMABlackwellSupport]>;
+// TMA Tensor Prefetch from Global memory with Override Global Address
+// (with cache hint and eviction priority forms).
+multiclass NVPTX_TMA_TENSOR_PREFETCH_OVERRIDE_INTR<int dim, string mode,
+ string override> {
+ defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
+ defvar num_dims = TMA_DIMS_UTIL<dim, mode>.num_dims;
+
+ defvar prefix = "cp.async.bulk.prefetch.tensor." # num_dims # "d.L2.global";
+ defvar suffix = "." # !subst("_", "::", mode) # base.override_suffix;
+ defvar asm_str = " [$tmap, $override_addr, " # base.tensor_size_component #
+ "{{" # base.dims_str # "}}]" # base.im2col_component;
+
+ defvar intr =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_" #
+ mode # "_" # override # "_" # num_dims # "d");
+ defvar intr_evict =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_" #
+ mode # "_" # override # "_" # num_dims # "d");
+
+ defvar ins_dag_base =
+ !con((ins B64:$tmap, B64:$override_addr),
+ base.tensor_size_dag, base.lower_stride_dag,
+ base.upper_stride_dag, base.dims_dag,
+ base.im2col_dag);
+
+ defvar ins_dag = ins_dag_base;
+ defvar ins_dag_with_ch = !con(ins_dag_base, (ins B64:$ch));
+ defvar ins_dag_evict = !con(ins_dag_base, (ins EvictPolicyOp:$policy));
+
+ defvar intr_dag_base =
+ !con((intr B64:$tmap, B64:$override_addr),
+ !setdagop(base.tensor_size_dag, intr),
+ !setdagop(base.lower_stride_dag, intr),
+ !setdagop(base.upper_stride_dag, intr),
+ !setdagop(base.dims_dag, intr),
+ !setdagop(base.im2col_dag, intr));
+
+ defvar intr_dag_evict_base =
+ !con((intr_evict B64:$tmap, B64:$override_addr),
+ !setdagop(base.tensor_size_dag, intr_evict),
+ !setdagop(base.lower_stride_dag, intr_evict),
+ !setdagop(base.upper_stride_dag, intr_evict),
+ !setdagop(base.dims_dag, intr_evict),
+ !setdagop(base.im2col_dag, intr_evict));
+
+ defvar intr_dag = !con(intr_dag_base, (intr (i64 srcvalue), 0));
+ defvar intr_dag_with_ch = !con(intr_dag_base, (intr B64:$ch, -1));
+ defvar intr_dag_evict = !con(intr_dag_evict_base,
+ (intr_evict PrefetchEvictPolicy:$policy));
+
+ let Predicates = [hasRubinFamilySupport] in {
+ def "" : NVPTXInst<(outs), ins_dag,
+ prefix # suffix # asm_str # ";",
+ [intr_dag]>;
+ def _CH : NVPTXInst<(outs), ins_dag_with_ch,
+ prefix # suffix # ".L2::cache_hint" # asm_str # ", $ch;",
+ [intr_dag_with_ch]>;
+ def _EV : NVPTXInst<(outs), ins_dag_evict,
+ prefix # suffix # "${policy}" # asm_str # ";",
+ [intr_dag_evict]>;
+ }
+}
+
+foreach dim = 1...5 in {
+ foreach mode = !if(!ge(dim, 3),
+ ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
+ defvar suffix = !toupper(mode # "_" # override) # "_" # dim # "D";
+ defm TMA_TENSOR_PFO_ # suffix
+ : NVPTX_TMA_TENSOR_PREFETCH_OVERRIDE_INTR<dim, mode, override>;
+ }
+ }
+}
+defm TMA_TENSOR_PFO_TILE_GATHER4_OVERRIDE_ADDR
+ : NVPTX_TMA_TENSOR_PREFETCH_OVERRIDE_INTR<5, "tile_gather4", "override_addr">;
+
+// TMA applypriority variants.
+multiclass NVPTX_APPLYPRIORITY_BULK_TENSOR_INTR<int dim, string mode> {
+ defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar asm_str_base = " [$tmap, {{" # dims_str # "}}]";
+
+ defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;
+ defvar im2col_str = TMA_IM2COL_UTIL<dim, mode>.base_str;
+ defvar asm_str = !if(!empty(im2col_str),
+ asm_str_base,
+ asm_str_base # ", {{" # im2col_str # "}}");
+
+ defvar dim_val = TMA_DIMS_UTIL<dim, mode>.num_dims;
+ defvar inst_name = "applypriority.async.bulk.tensor" # "." # dim_val # "d"
+ # ".global.bulk_group.L2::evict_normal"
+ # "." # !subst("_", "::", mode);
+
+ defvar intr =
+ !cast<Intrinsic>("int_nvvm_applypriority_async_bulk_tensor_evict_priority_" #
+ mode # "_" # dim_val # "d");
+
+ defvar ins_dag = !con((ins B64:$tmap),
+ dims_dag,
+ im2col_dag);
+ defvar intr_dag = !con((intr B64:$tmap),
+ !setdagop(dims_dag, intr),
+ !setdagop(im2col_dag, intr),
+ (intr 0));
+
+ def "" : NVPTXInst<(outs), ins_dag,
+ inst_name # asm_str # ";",
+ [intr_dag]>,
+ Requires<[hasRubinFamilySupport]>;
+}
+
+foreach dim = 1...5 in {
+ foreach mode = !if(!ge(dim, 3),
+ ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ defvar suffix = !toupper(mode) # "_" # dim # "D";
+ defm NVPTX_APPLYPRIORITY_BULK_TENSOR_ # suffix
+ : NVPTX_APPLYPRIORITY_BULK_TENSOR_INTR<dim, mode>;
+ }
+}
+defm NVPTX_APPLYPRIORITY_BULK_TENSOR_TILE_GATHER4_2D
+ : NVPTX_APPLYPRIORITY_BULK_TENSOR_INTR<5, "tile_gather4">;
+
+// TMA applypriority variants with override global address.
+multiclass NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<int dim, string mode,
+ string override> {
+ defvar base = TMA_TENSOR_OVERRIDE_BASE<dim, mode, override>;
+ defvar num_dims = TMA_DIMS_UTIL<dim, mode>.num_dims;
+
+ defvar prefix = "applypriority.async.bulk.tensor" # "." # num_dims # "d"
+ # ".global.bulk_group.L2::evict_normal";
+ defvar suffix = "." # !subst("_", "::", mode) # base.override_suffix;
+ defvar asm_str = " [$tmap, $override_addr, " # base.tensor_size_component #
+ "{{" # base.dims_str # "}}]" # base.im2col_component;
+
+ defvar intr =
+ !cast<Intrinsic>("int_nvvm_applypriority_async_bulk_tensor_evict_priority_" #
+ mode # "_" # override # "_" # num_dims # "d");
+
+ defvar ins_dag =
+ !con((ins B64:$tmap, B64:$override_addr),
+ base.tensor_size_dag, base.lower_stride_dag,
+ base.upper_stride_dag, base.dims_dag,
+ base.im2col_dag);
+
+ defvar intr_dag =
+ !con((intr B64:$tmap, B64:$override_addr),
+ !setdagop(base.tensor_size_dag, intr),
+ !setdagop(base.lower_stride_dag, intr),
+ !setdagop(base.upper_stride_dag, intr),
+ !setdagop(base.dims_dag, intr),
+ !setdagop(base.im2col_dag, intr),
+ (intr 0));
+
+ def "" : NVPTXInst<(outs), ins_dag,
+ prefix # suffix # asm_str # ";",
+ [intr_dag]>,
+ Requires<[hasRubinFamilySupport]>;
+}
+
+foreach dim = 1...5 in {
+ foreach mode = !if(!ge(dim, 3),
+ ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
+ defvar suffix = !toupper(mode # "_" # override) # "_" # dim # "D";
+ defm NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_ # suffix
+ : NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<dim, mode, override>;
+ }
+ }
+}
+defm NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_TILE_GATHER4_OVERRIDE_ADDR
+ : NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<5, "tile_gather4", "override_addr">;
+
+
+
//Prefetchu and Prefetch
class PREFETCH_TENSORMAP_INST<NVPTXAddressSpace as>
diff --git a/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-global-evict-priority.ll b/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-global-evict-priority.ll
new file mode 100644
index 0000000000000..a7bcff5b1894e
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-global-evict-priority.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -target-abi=shortptr | FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1), i32, i32)
+
+define void @applypriority_async_bulk_evict_priority(ptr addrspace(1) %src, i32 %sz) {
+; CHECK-LABEL: applypriority_async_bulk_evict_priority(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [applypriority_async_bulk_evict_priority_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [applypriority_async_bulk_evict_priority_param_1];
+; CHECK-NEXT: applypriority.async.bulk.global.bulk_group.L2::evict_normal [%rd1], %r1;
+; CHECK-NEXT: mov.b32 %r2, 128;
+; CHECK-NEXT: applypriority.async.bulk.global.bulk_group.L2::evict_normal [%rd1], %r2;
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: applypriority_async_bulk_evict_priority(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<3>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [applypriority_async_bulk_evict_priority_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [applypriority_async_bulk_evict_priority_param_1];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.global.bulk_group.L2::evict_normal [%rd1], %r1;
+; CHECK-PTX-SHARED32-NEXT: mov.b32 %r2, 128;
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.global.bulk_group.L2::evict_normal [%rd1], %r2;
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1) readonly captures(none), i32, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT-LABEL: define void @applypriority_async_bulk_evict_priority(ptr addrspace(1) %src, i32 %sz) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1) %src, i32 %sz, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1) %src, i32 128, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1) %src, i32 %sz, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1) %src, i32 128, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-evict-priority.ll b/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-evict-priority.ll
new file mode 100644
index 0000000000000..18150873833a2
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-evict-priority.ll
@@ -0,0 +1,252 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -target-abi=shortptr | FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.1d(ptr readonly, i32, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.2d(ptr readonly, i32, i32, i32, i32, i32, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.5d(ptr readonly, i32, i32, i32, i32, i32, i16, i16, i16, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.5d(ptr readonly, i32, i32, i32, i32, i32, i16, i16, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.5d(ptr readonly, i32, i32, i32, i32, i32, i16, i16, i32 immarg range(i32 0, 1)) #0
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.1d(ptr, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.2d(ptr, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.3d(ptr, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.4d(ptr, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.5d(ptr, i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.2d(ptr, i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.3d(ptr, i32, i32, i32, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.4d(ptr, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.5d(ptr, i32, i32, i32, i32, i32, i16, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.3d(ptr, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.4d(ptr, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.5d(ptr, i32, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.3d(ptr, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.4d(ptr, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.5d(ptr, i32, i32, i32, i32, i32, i16, i16, i32)
+
+define void @tile(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_param_1];
+; CHECK-NEXT: applypriority.async.bulk.tensor.1d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1}];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_param_2];
+; CHECK-NEXT: applypriority.async.bulk.tensor.2d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2}];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_param_3];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2, %r3}];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_param_4];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2, %r3, %r4}];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: tile(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [tile_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [tile_param_1];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.1d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [tile_param_2];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.2d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [tile_param_3];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2, %r3}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [tile_param_4];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2, %r3, %r4}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [tile_param_5];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.tile [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.1d(ptr %tmap, i32 %d0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.2d(ptr %tmap, i32 %d0, i32 %d1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.1d(ptr %tmap, i32 %d0, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.2d(ptr %tmap, i32 %d0, i32 %d1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 0)
+ ret void
+}
+
+define void @gather4(ptr %tmap, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-LABEL: gather4(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [gather4_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [gather4_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [gather4_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [gather4_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r4, [gather4_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r5, [gather4_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.2d.global.bulk_group.L2::evict_normal.tile::gather4 [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @gather4(ptr %tmap, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.2d(ptr %tmap, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.2d(ptr %tmap, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 0)
+ ret void
+}
+
+define void @im2col(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2) {
+; CHECK-LABEL: im2col(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_param_6];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col [%rd1, {%r1, %r2, %r3}], {%rs1};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_param_7];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_param_5];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [im2col_param_8];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: im2col(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b16 %rs<4>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [im2col_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [im2col_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [im2col_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [im2col_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs1, [im2col_param_6];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col [%rd1, {%r1, %r2, %r3}], {%rs1};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [im2col_param_4];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs2, [im2col_param_7];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [im2col_param_5];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs3, [im2col_param_8];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, i32 0)
+ ret void
+}
+
+define void @im2col_w(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-LABEL: im2col_w(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_param_7];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col::w [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_param_4];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col::w [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col::w [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: im2col_w(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b16 %rs<3>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [im2col_w_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [im2col_w_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [im2col_w_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [im2col_w_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs1, [im2col_w_param_6];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs2, [im2col_w_param_7];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col::w [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [im2col_w_param_4];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col::w [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [im2col_w_param_5];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col::w [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 0)
+ ret void
+}
+
+define void @im2col_w_128(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-LABEL: im2col_w_128(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_128_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_128_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_128_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_128_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_128_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_128_param_7];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col::w::128 [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_128_param_4];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_128_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: im2col_w_128(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b16 %rs<3>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [im2col_w_128_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [im2col_w_128_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [im2col_w_128_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [im2col_w_128_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs1, [im2col_w_128_param_6];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs2, [im2col_w_128_param_7];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col::w::128 [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [im2col_w_128_param_4];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [im2col_w_128_param_5];
+; CHECK-PTX-SHARED32-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w_128(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-override-evict-priority.ll b/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-override-evict-priority.ll
new file mode 100644
index 0000000000000..bd29862e05416
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/applypriority-async-bulk-tensor-override-evict-priority.ll
@@ -0,0 +1,225 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.1d(ptr readonly, ptr addrspace(1) readonly, i32, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.5d(ptr readonly, ptr addrspace(1) readonly, i16, i16, i16, i16, i16, i32, i32, i32, i32, i16, i32, i32, i32, i32, i32, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.override.addr.2d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.5d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i16, i16, i16, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.5d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i16, i16, i32 immarg range(i32 0, 1)) #0
+; CHECK-FORMAT: declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.5d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i16, i16, i32 immarg range(i32 0, 1)) #0
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.1d(ptr, ptr addrspace(1), i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.2d(ptr, ptr addrspace(1), i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.1d(ptr, ptr addrspace(1), i16, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.2d(ptr, ptr addrspace(1), i16, i16, i32, i16, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.3d(ptr, ptr addrspace(1), i16, i16, i16, i32, i32, i16, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.4d(ptr, ptr addrspace(1), i16, i16, i16, i16, i32, i32, i32, i16, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.5d(ptr, ptr addrspace(1), i16, i16, i16, i16, i16, i32, i32, i32, i32, i16, i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.override.addr.2d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i32)
+
+define void @tile_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_override_addr_param_2];
+; CHECK-NEXT: applypriority.async.bulk.tensor.1d.global.bulk_group.L2::evict_normal.tile.override::global_address [%rd1, %rd2, {%r1}];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_override_addr_param_3];
+; CHECK-NEXT: applypriority.async.bulk.tensor.2d.global.bulk_group.L2::evict_normal.tile.override::global_address [%rd1, %rd2, {%r1, %r2}];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_override_addr_param_4];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.tile.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_override_addr_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.tile.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_override_addr_param_6];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.tile.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 0)
+ ret void
+}
+
+define void @tile_override_addr_dim_stride(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile_override_addr_dim_stride(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: applypriority.async.bulk.tensor.1d.global.bulk_group.L2::evict_normal.tile.override::global_address.override::global_dim [%rd1, %rd2, {%rs1}, {%r1}];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: applypriority.async.bulk.tensor.2d.global.bulk_group.L2::evict_normal.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}];
+; CHECK-NEXT: ld.param::func.b16 %rs4, [tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}];
+; CHECK-NEXT: ld.param::func.b16 %rs5, [tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param::func.b32 %r7, [tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}];
+; CHECK-NEXT: ld.param::func.b16 %rs6, [tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r8, [tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param::func.b32 %r9, [tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_override_addr_dim_stride(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 0)
+ ret void
+}
+
+define void @gather4_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-LABEL: gather4_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [gather4_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [gather4_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [gather4_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [gather4_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [gather4_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r4, [gather4_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r5, [gather4_override_addr_param_6];
+; CHECK-NEXT: applypriority.async.bulk.tensor.2d.global.bulk_group.L2::evict_normal.tile::gather4.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @gather4_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 0)
+ ret void
+}
+
+define void @im2col_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2) {
+; CHECK-LABEL: im2col_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_override_addr_param_7];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_override_addr_param_8];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [im2col_override_addr_param_9];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %i0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, i32 0)
+ ret void
+}
+
+define void @im2col_w_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-LABEL: im2col_w_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_override_addr_param_8];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col::w.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_override_addr_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col::w.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_override_addr_param_6];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col::w.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 0)
+ ret void
+}
+
+define void @im2col_w_128_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-LABEL: im2col_w_128_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_128_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_w_128_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_128_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_128_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_128_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_128_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_128_override_addr_param_8];
+; CHECK-NEXT: applypriority.async.bulk.tensor.3d.global.bulk_group.L2::evict_normal.im2col::w::128.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_128_override_addr_param_5];
+; CHECK-NEXT: applypriority.async.bulk.tensor.4d.global.bulk_group.L2::evict_normal.im2col::w::128.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_128_override_addr_param_6];
+; CHECK-NEXT: applypriority.async.bulk.tensor.5d.global.bulk_group.L2::evict_normal.im2col::w::128.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w_128_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-prefetch-evict-priority.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-prefetch-evict-priority.ll
new file mode 100644
index 0000000000000..fef6299365687
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-prefetch-evict-priority.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -target-abi=shortptr | FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1), i32, i32)
+
+define void @cp_async_bulk_prefetch_evict_priority(ptr addrspace(1) %src, i32 %sz) {
+; CHECK-LABEL: cp_async_bulk_prefetch_evict_priority(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_prefetch_evict_priority_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_prefetch_evict_priority_param_1];
+; CHECK-NEXT: cp.async.bulk.prefetch.L2.global [%rd1], %r1;
+; CHECK-NEXT: cp.async.bulk.prefetch.L2.global.L2::evict_last [%rd1], %r1;
+; CHECK-NEXT: mov.b32 %r2, 128;
+; CHECK-NEXT: cp.async.bulk.prefetch.L2.global [%rd1], %r2;
+; CHECK-NEXT: cp.async.bulk.prefetch.L2.global.L2::evict_last [%rd1], %r2;
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: cp_async_bulk_prefetch_evict_priority(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<3>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_prefetch_evict_priority_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_prefetch_evict_priority_param_1];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.L2.global [%rd1], %r1;
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.L2.global.L2::evict_last [%rd1], %r1;
+; CHECK-PTX-SHARED32-NEXT: mov.b32 %r2, 128;
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.L2.global [%rd1], %r2;
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.L2.global.L2::evict_last [%rd1], %r2;
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_prefetch_evict_priority(ptr addrspace(1) %src, i32 %sz) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 %sz, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 %sz, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 128, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 128, /* evict_policy=L2::evict_last */ i32 1)
+
+ tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 %sz, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 %sz, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 128, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %src, i32 128, i32 1)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-evict-priority.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-evict-priority.ll
new file mode 100644
index 0000000000000..a6fcd64ee1c3f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-evict-priority.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -target-abi=shortptr | FileCheck --check-prefixes=CHECK-PTX-SHARED32 %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.1d(ptr, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.2d(ptr, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.3d(ptr, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.4d(ptr, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.5d(ptr, i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.3d(ptr, i32, i32, i32, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.4d(ptr, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.5d(ptr, i32, i32, i32, i32, i32, i16, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.3d(ptr, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.4d(ptr, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.5d(ptr, i32, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.3d(ptr, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.4d(ptr, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.5d(ptr, i32, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.2d(ptr, i32, i32, i32, i32, i32, i32)
+
+define void @tile(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_param_1];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile [%rd1, {%r1}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.L2::evict_last [%rd1, {%r1}];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_param_2];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile [%rd1, {%r1, %r2}];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_param_3];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2}];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_param_4];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile [%rd1, {%r1, %r2, %r3}];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2, %r3}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile [%rd1, {%r1, %r2, %r3, %r4}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: tile(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [tile_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [tile_param_1];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile [%rd1, {%r1}];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.L2::evict_last [%rd1, {%r1}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [tile_param_2];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile [%rd1, {%r1, %r2}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [tile_param_3];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [tile_param_4];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile [%rd1, {%r1, %r2, %r3}];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [tile_param_5];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2, %r3}];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile [%rd1, {%r1, %r2, %r3, %r4}];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.1d(ptr %tmap, i32 %d0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.1d(ptr %tmap, i32 %d0, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.2d(ptr %tmap, i32 %d0, i32 %d1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.2d(ptr %tmap, i32 %d0, i32 %d1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_last */ i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.1d(ptr %tmap, i32 %d0, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.1d(ptr %tmap, i32 %d0, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.2d(ptr %tmap, i32 %d0, i32 %d1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.2d(ptr %tmap, i32 %d0, i32 %d1, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 1)
+ ret void
+}
+
+define void @im2col(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2) {
+; CHECK-LABEL: im2col(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col [%rd1, {%r1, %r2, %r3}], {%rs1};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col.L2::evict_last [%rd1, {%r1, %r2, %r3}], {%rs1};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_param_7];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_param_5];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [im2col_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: im2col(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b16 %rs<4>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [im2col_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [im2col_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [im2col_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [im2col_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs1, [im2col_param_6];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col [%rd1, {%r1, %r2, %r3}], {%rs1};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col.L2::evict_last [%rd1, {%r1, %r2, %r3}], {%rs1};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [im2col_param_4];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs2, [im2col_param_7];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [im2col_param_5];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs3, [im2col_param_8];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, /* evict_policy=L2::evict_last */ i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i16 %i2, i32 1)
+ ret void
+}
+
+define void @im2col_w(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-LABEL: im2col_w(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_param_7];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w.L2::evict_last [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_param_4];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: im2col_w(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b16 %rs<3>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [im2col_w_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [im2col_w_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [im2col_w_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [im2col_w_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs1, [im2col_w_param_6];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs2, [im2col_w_param_7];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w.L2::evict_last [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [im2col_w_param_4];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [im2col_w_param_5];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 1)
+ ret void
+}
+
+define void @im2col_w_128(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-LABEL: im2col_w_128(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_128_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_128_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_128_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_128_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_128_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_128_param_7];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128 [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128.L2::evict_last [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_128_param_4];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_128_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: im2col_w_128(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b16 %rs<3>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [im2col_w_128_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [im2col_w_128_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [im2col_w_128_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [im2col_w_128_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs1, [im2col_w_128_param_6];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b16 %rs2, [im2col_w_128_param_7];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128 [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128.L2::evict_last [%rd1, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [im2col_w_128_param_4];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [im2col_w_128_param_5];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128 [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w_128(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, /* evict_policy=L2::evict_last */ i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %i0, i16 %i1, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %i0, i16 %i1, i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %i0, i16 %i1, i32 1)
+ ret void
+}
+
+define void @tile_gather4(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile_gather4(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_gather4_param_0];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_gather4_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_gather4_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_gather4_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_gather4_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_gather4_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4 [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-PTX-SHARED32-LABEL: tile_gather4(
+; CHECK-PTX-SHARED32: {
+; CHECK-PTX-SHARED32-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-SHARED32-NEXT: .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT: // %bb.0:
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b64 %rd1, [tile_gather4_param_0];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r1, [tile_gather4_param_1];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r2, [tile_gather4_param_2];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r3, [tile_gather4_param_3];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r4, [tile_gather4_param_4];
+; CHECK-PTX-SHARED32-NEXT: ld.param::func.b32 %r5, [tile_gather4_param_5];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4 [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-PTX-SHARED32-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.L2::evict_last [%rd1, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-PTX-SHARED32-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_gather4(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.2d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.2d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_last */ i32 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.2d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.2d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 1)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override-evict-priority.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override-evict-priority.ll
new file mode 100644
index 0000000000000..c8851862350f4
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override-evict-priority.ll
@@ -0,0 +1,245 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr readonly, ptr addrspace(1) readonly, i32, i32 immarg range(i32 0, 2))
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.5d(ptr readonly, ptr addrspace(1) readonly, i16, i16, i16, i16, i16, i32, i32, i32, i32, i16, i32, i32, i32, i32, i32, i32 immarg range(i32 0, 2))
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.5d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i16, i16, i16, i32 immarg range(i32 0, 2))
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i32 immarg range(i32 0, 2))
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr, ptr addrspace(1), i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.2d(ptr, ptr addrspace(1), i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.1d(ptr, ptr addrspace(1), i16, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.2d(ptr, ptr addrspace(1), i16, i16, i32, i16, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.3d(ptr, ptr addrspace(1), i16, i16, i16, i32, i32, i16, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.4d(ptr, ptr addrspace(1), i16, i16, i16, i16, i32, i32, i32, i16, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.5d(ptr, ptr addrspace(1), i16, i16, i16, i16, i16, i32, i32, i32, i32, i16, i32, i32, i32, i32, i32, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i32)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i32)
+
+define void @tile_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_override_addr_param_2];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address.L2::evict_last [%rd1, %rd2, {%r1}];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_override_addr_param_3];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2}];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_override_addr_param_4];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3}];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4}];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1, %r2}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, /* evict_policy=L2::evict_normal */ i32 0)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, /* evict_policy=L2::evict_normal */ i32 0)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 1)
+
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 0)
+
+ ret void
+}
+
+define void @tile_override_addr_dim_stride(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-LABEL: tile_override_addr_dim_stride(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address.override::global_dim.L2::evict_last [%rd1, %rd2, {%rs1}, {%r1}];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address.override::global_dim_stride.L2::evict_last [%rd1, %rd2, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}];
+; CHECK-NEXT: ld.param::func.b16 %rs4, [tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.override::global_address.override::global_dim_stride.L2::evict_last [%rd1, %rd2, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}];
+; CHECK-NEXT: ld.param::func.b16 %rs5, [tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param::func.b32 %r7, [tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.override::global_address.override::global_dim_stride.L2::evict_last [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}];
+; CHECK-NEXT: ld.param::func.b16 %rs6, [tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r8, [tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param::func.b32 %r9, [tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.override::global_address.override::global_dim_stride.L2::evict_last [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_override_addr_dim_stride(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4) {
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, /* evict_policy=L2::evict_last */ i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i32 1)
+ ret void
+}
+
+define void @im2col_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2) {
+; CHECK-LABEL: im2col_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_override_addr_param_7];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [im2col_override_addr_param_9];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2) {
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, /* evict_policy=L2::evict_last */ i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i32 1)
+ ret void
+}
+
+define void @im2col_w_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset) {
+; CHECK-LABEL: im2col_w_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset) {
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, /* evict_policy=L2::evict_last */ i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i32 1)
+ ret void
+}
+
+define void @im2col_w128_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset) {
+; CHECK-LABEL: im2col_w128_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w128_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_w128_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w128_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w128_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w128_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w128_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w128_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w128_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w128_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w128_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset) {
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, /* evict_policy=L2::evict_last */ i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i32 1)
+ ret void
+}
+
+define void @tile_gather4_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-LABEL: tile_gather4_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_gather4_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_gather4_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_gather4_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_gather4_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_gather4_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_gather4_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_gather4_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.override::global_address.L2::evict_last [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_gather4_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, /* evict_policy=L2::evict_last */ i32 1)
+; CHECK-FORMAT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, /* evict_policy=L2::evict_normal */ i32 0)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i32 0)
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override.ll
new file mode 100644
index 0000000000000..3fa672fcc38ff
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch-override.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr readonly, ptr addrspace(1) readonly, i32, i64, i1 immarg)
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(ptr readonly, ptr addrspace(1) readonly, i16, i16, i16, i16, i16, i32, i32, i32, i32, i16, i32, i32, i32, i32, i32, i64, i1 immarg)
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i16, i16, i16, i64, i1 immarg)
+; CHECK-FORMAT: declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr readonly, ptr addrspace(1) readonly, i32, i32, i32, i32, i32, i64, i1 immarg)
+
+target triple = "nvptx64-nvidia-cuda"
+
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr, ptr addrspace(1), i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.2d(ptr, ptr addrspace(1), i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.1d(ptr, ptr addrspace(1), i16, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.2d(ptr, ptr addrspace(1), i16, i16, i32, i16, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.3d(ptr, ptr addrspace(1), i16, i16, i16, i32, i32, i16, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.4d(ptr, ptr addrspace(1), i16, i16, i16, i16, i32, i32, i32, i16, i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(ptr, ptr addrspace(1), i16, i16, i16, i16, i16, i32, i32, i32, i32, i16, i32, i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.3d(ptr, ptr addrspace(1), i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.4d(ptr, ptr addrspace(1), i32, i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.5d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i16, i16, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr, ptr addrspace(1), i32, i32, i32, i32, i32, i64, i1)
+
+define void @tile_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i64 %ch) {
+; CHECK-LABEL: tile_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_override_addr_param_2];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1}];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_override_addr_param_3];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1, %r2}];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_override_addr_param_4];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [tile_override_addr_param_17];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], %rd3;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.1d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+ ret void
+}
+
+define void @tile_override_addr_dim_stride(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-LABEL: tile_override_addr_dim_stride(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_override_addr_dim_stride_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_override_addr_dim_stride_param_1];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [tile_override_addr_dim_stride_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_override_addr_dim_stride_param_12];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address.override::global_dim [%rd1, %rd2, {%rs1}, {%r1}];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [tile_override_addr_dim_stride_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_override_addr_dim_stride_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [tile_override_addr_dim_stride_param_11];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_override_addr_dim_stride_param_13];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}];
+; CHECK-NEXT: ld.param::func.b16 %rs4, [tile_override_addr_dim_stride_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_override_addr_dim_stride_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_override_addr_dim_stride_param_14];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}];
+; CHECK-NEXT: ld.param::func.b16 %rs5, [tile_override_addr_dim_stride_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tile_override_addr_dim_stride_param_9];
+; CHECK-NEXT: ld.param::func.b32 %r7, [tile_override_addr_dim_stride_param_15];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}];
+; CHECK-NEXT: ld.param::func.b16 %rs6, [tile_override_addr_dim_stride_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r8, [tile_override_addr_dim_stride_param_10];
+; CHECK-NEXT: ld.param::func.b32 %r9, [tile_override_addr_dim_stride_param_16];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.override::global_address.override::global_dim_stride [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [tile_override_addr_dim_stride_param_17];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.1d.L2.global.tile.override::global_address.override::global_dim.L2::cache_hint [%rd1, %rd2, {%rs1}, {%r1}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile.override::global_address.override::global_dim_stride.L2::cache_hint [%rd1, %rd2, {%rs1, %rs2}, {%r2}, %rs3, {%r1, %r3}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.tile.override::global_address.override::global_dim_stride.L2::cache_hint [%rd1, %rd2, {%rs1, %rs2, %rs4}, {%r2, %r4}, %rs3, {%r1, %r3, %r5}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.tile.override::global_address.override::global_dim_stride.L2::cache_hint [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5}, {%r2, %r4, %r6}, %rs3, {%r1, %r3, %r5, %r7}], %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.tile.override::global_address.override::global_dim_stride.L2::cache_hint [%rd1, %rd2, {%rs1, %rs2, %rs4, %rs5, %rs6}, {%r2, %r4, %r6, %r8}, %rs3, {%r1, %r3, %r5, %r7, %r9}], %rd3;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_override_addr_dim_stride(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* flag_cache_hint= */ i1 true)
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.1d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.2d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %lstrd0, i16 %ustrd, i32 %d0, i32 %d1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.3d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i32 %lstrd0, i32 %lstrd1, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.4d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.stride.5d(ptr %tmap, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %lstrd0, i32 %lstrd1, i32 %lstrd2, i32 %lstrd3, i16 %ustrd, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+
+ ret void
+}
+
+define void @im2col_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch) {
+; CHECK-LABEL: im2col_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_override_addr_param_7];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_override_addr_param_6];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [im2col_override_addr_param_9];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
+; CHECK-NEXT: ld.param::func.b64 %rd3, [im2col_override_addr_param_10];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1}, %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3}, %rd3;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch, /* flag_cache_hint= */ i1 true)
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch, i1 1)
+
+ ret void
+}
+
+define void @im2col_w_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch) {
+; CHECK-LABEL: im2col_w_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_w_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b64 %rd3, [im2col_w_override_addr_param_9];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 true)
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 1)
+
+ ret void
+}
+
+define void @im2col_w128_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch) {
+; CHECK-LABEL: im2col_w128_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [im2col_w128_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [im2col_w128_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [im2col_w128_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [im2col_w128_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [im2col_w128_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [im2col_w128_override_addr_param_7];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [im2col_w128_override_addr_param_8];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128.override::global_address [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r4, [im2col_w128_override_addr_param_5];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b32 %r5, [im2col_w128_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b64 %rd3, [im2col_w128_override_addr_param_9];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.3d.L2.global.im2col::w::128.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.4d.L2.global.im2col::w::128.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.5d.L2.global.im2col::w::128.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2}, %rd3;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @im2col_w128_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, /* flag_cache_hint= */ i1 true)
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 0)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.3d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.4d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 1)
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.override.addr.5d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2colHalo, i16 %im2colOffset, i64 %ch, i1 1)
+
+ ret void
+}
+
+define void @tile_gather4_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch) {
+; CHECK-LABEL: tile_gather4_override_addr(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tile_gather4_override_addr_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tile_gather4_override_addr_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tile_gather4_override_addr_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tile_gather4_override_addr_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tile_gather4_override_addr_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tile_gather4_override_addr_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tile_gather4_override_addr_param_6];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.override::global_address [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [tile_gather4_override_addr_param_7];
+; CHECK-NEXT: cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.override::global_address.L2::cache_hint [%rd1, %rd2, {%r1, %r2, %r3, %r4, %r5}], %rd3;
+; CHECK-NEXT: ret;
+;
+; CHECK-FORMAT-LABEL: define void @tile_gather4_override_addr(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, /* flag_cache_hint= */ i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, /* flag_cache_hint= */ i1 true)
+ ; without cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 0)
+
+ ; with cache hint
+ tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.override.addr.2d(ptr %tmap, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 1)
+
+ ret void
+}
+
+
>From b2614c07b23208bc231a4ed3cba887fa1f46ca8d Mon Sep 17 00:00:00 2001
From: Rajat Bajpai <rbajpai at nvidia.com>
Date: Sat, 22 Aug 2026 19:49:15 +0000
Subject: [PATCH 2/2] Addressed review comments
---
llvm/docs/NVPTXUsage.md | 84 +++++++++--------
llvm/include/llvm/IR/IntrinsicsNVVM.td | 109 +++++++++++++----------
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 62 +++++--------
3 files changed, 129 insertions(+), 126 deletions(-)
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index 9802fb6af638d..5f6a00945a666 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1627,11 +1627,11 @@ Tensor intrinsics with `override` in their name replace properties in the
opaque tensor map with explicit operands. The supported override variants and
operand layouts are:
-| Intrinsic suffix | Modes and dimensions | Operands after `%tensor_map` |
-|------------------|----------------------|------------------------------|
-| `override.addr` | `tile` 1D--5D; `im2col`, `im2col.w`, and `im2col.w.128` 3D--5D; `tile.gather4` or `tile.scatter4` 2D | `ptr addrspace(1) %override_addr` |
-| `override.addr.dim` | `tile` 1D | `%override_addr, i16 %ts0` |
-| `override.addr.dim.stride` | `tile` 2D--5D | `%override_addr, i16 %ts0 ... i16 %ts{N-1}, i32 %stride0 ... i32 %stride{N-2}, i16 %upper_stride` |
+| Intrinsic suffix | Modes and dimensions | Operands after `%tensor_map` |
+|----------------------------|------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|
+| `override.addr` | `tile` 1D--5D; `im2col`, `im2col.w`, and `im2col.w.128` 3D--5D; `tile.gather4` or `tile.scatter4` 2D | `ptr addrspace(1) %override_addr` |
+| `override.addr.dim` | `tile` 1D | `%override_addr, i16 %ts0` |
+| `override.addr.dim.stride` | `tile` 2D--5D | `%override_addr, i16 %ts0 ... i16 %ts{N-1}, i32 %stride0 ... i32 %stride{N-2}, i16 %upper_stride` |
`%override_addr` replaces the tensor map's global base address. It must be
16-byte aligned; otherwise a runtime error is raised. The 128 KiB range
@@ -1650,8 +1650,7 @@ These variants lower to `.override::global_address`, optionally followed by
`.override::global_dim` or `.override::global_dim_stride`, and require
`sm_107f` and PTX 9.4 or later.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-overriding-tensor-property-value).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-overriding-tensor-property-value).
#### '`llvm.nvvm.cp.async.bulk.global.to.shared.cluster`'
@@ -1784,9 +1783,10 @@ declare void @llvm.nvvm.cp.async.bulk.prefetch.evict.priority(ptr addrspace(1) %
The '`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`' intrinsic
asynchronously prefetches `%size` bytes from global memory into the L2 cache
using the selected eviction priority. It lowers to the
-`cp.async.bulk.prefetch.L2.global` PTX instruction with an L2 eviction-priority
-qualifier. Unlike `@llvm.nvvm.cp.async.bulk.prefetch.L2`, it takes an eviction
-policy instead of a cache hint. `%size` must be a multiple of 16.
+`cp.async.bulk.prefetch.L2.global` PTX instruction with an L2
+eviction-priority qualifier. Unlike `@llvm.nvvm.cp.async.bulk.prefetch.L2`,
+it takes an eviction policy instead of a cache hint. `%size` must be a
+multiple of 16.
The trailing `i32 %evict_policy` is an immediate (`immarg`) argument that
selects the L2 eviction policy.
@@ -1816,13 +1816,12 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.gather
These intrinsics asynchronously prefetch tile-mode tensor data from global
memory into the L2 cache using the selected eviction priority. They are the
eviction-priority forms of the corresponding
-`@llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.*` intrinsics, and the trailing
-`i32 %evict_policy` uses the encoding described for
+`@llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.*` intrinsics, and the
+trailing `i32 %evict_policy` uses the encoding described for
`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`. They do not take cache-hint
operands and require `sm_107f` and PTX 9.4 or later.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.[3-5]d`'
@@ -1854,8 +1853,7 @@ The trailing `i32 %evict_policy` uses the encoding described for
`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`. These forms do not take
cache-hint operands and require `sm_107f` and PTX 9.4 or later.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override*.[1-5]d`'
@@ -1880,19 +1878,17 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.override.addr.dim.str
These intrinsics asynchronously prefetch tile-mode tensor data from global
memory into the L2 cache while overriding tensor-map properties with explicit
operands. They lower to the
-`cp.async.bulk.prefetch.tensor.<N>d.L2.global.tile` PTX instructions qualified
-with `.override::*`.
+`cp.async.bulk.prefetch.tensor.<N>d.L2.global.tile` PTX instructions
+qualified with `.override::*`.
The supported override variants and operands are described in
[Tensor-Map Property Overrides](#tensor-map-property-overrides).
- The last argument is a compile-time boolean indicating whether `%cache_hint`
- is valid. When set, it
- indicates a valid cache_hint (`i64 %ch`) and generates the
+ is valid. When set, it indicates a valid cache_hint (`i64 %ch`) and generates the
`.L2::cache_hint` variant of the PTX instruction.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.override.addr.[3-5]d`'
@@ -1919,8 +1915,7 @@ N - 2 `i16` im2col offsets, while the `im2col.w` and `im2col.w.128` forms take
`i1 %flag_cache_hint` must be a compile-time constant; when set, it marks
`i64 %cache_hint` as valid and emits the `.L2::cache_hint` PTX qualifier.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.tile.override*.[1-5]d`'
@@ -1955,8 +1950,7 @@ The trailing `i32 %evict_policy` is an immediate argument: 0 selects the
default `evict_normal` policy and emits no PTX qualifier; 1 selects
`.L2::evict_last`. These forms do not take cache-hint operands.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
#### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.evict.priority.im2col.override.addr.[3-5]d`'
@@ -1984,8 +1978,7 @@ coordinates. The `im2col` forms take N - 2 `i16` im2col offsets, while the
`@llvm.nvvm.cp.async.bulk.prefetch.evict.priority`; these forms do not take
cache-hint operands.
-For more information, refer to the
-[PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
+For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
#### '`llvm.nvvm.prefetch.*`'
@@ -2062,11 +2055,10 @@ declare void @llvm.nvvm.applypriority.async.bulk.evict.priority(ptr addrspace(1)
##### Overview:
The '`@llvm.nvvm.applypriority.async.bulk.evict.priority`' intrinsic applies
-the selected L2 eviction priority to
-\[%global_ptr, %global_ptr + %size). It lowers to the
-`applypriority.async.bulk.global.bulk_group` PTX instruction. `%size` may be a
-register or an immediate and must be a multiple of 16. `%global_ptr` must be
-aligned to 128 bytes.
+the selected L2 eviction priority to `[%global_ptr, %global_ptr + %size)`.
+It lowers to the `applypriority.async.bulk.global.bulk_group` PTX instruction.
+`%size` may be a register or an immediate and must be a multiple of 16.
+`%global_ptr` must be aligned to 128 bytes.
`%evict_policy` is an immediate argument. Its only valid value is 0, which
selects `.L2::evict_normal`.
@@ -2104,7 +2096,6 @@ declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.4d
declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.5d(..., i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
declare void @llvm.nvvm.applypriority.async.bulk.tensor.evict.priority.im2col.w.128.{3,4,5}d(..., i16 %wHalo, i16 %wOffset, ...)
-
```
##### Overview:
@@ -2653,7 +2644,8 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.scatter4.override.addr.2d(
The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.tile.override*.[1-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.[1-5]d.*` set of PTX instructions qualified with the `.override::*` qualifiers.
These instructions initiate an asynchronous copy of tensor data from shared::cta to global memory (indicated by the `s2g` prefix) in `tile` mode, while overriding specific properties of the opaque tensor-map object with explicit instruction operands.
-The `.override::*` qualifiers ignore certain tensor-map properties and instead take them as explicit instruction operands.
+The `.override::*` qualifiers ignore certain tensor-map properties and
+instead take them as explicit instruction operands.
The supported override variants, operands, and restrictions are described in
[Tensor-Map Property Overrides](#tensor-map-property-overrides).
@@ -2681,9 +2673,11 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.5d(..., i3
##### Overview:
The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.override.addr.[3-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.im2col_no_offs.[3-5]d.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
-They copy tensor data from shared::cta to global memory in `im2col` mode using
-the `override.addr` operands described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
-The last boolean flag argument has the same functionality as described in the `s2g.tile` override intrinsics above.
+They copy tensor data from shared::cta to global memory in `im2col` mode
+using the `override.addr` operands described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
+The last boolean flag argument has the same functionality as described in the
+`s2g.tile` override intrinsics above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
@@ -2700,9 +2694,11 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.5d(...,
##### Overview:
The '`@llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.w.override.addr.[3-5]d`' intrinsics correspond to the `cp.async.bulk.tensor.im2col_no_offs::w.[3-5]d.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
-They copy tensor data from shared::cta to global memory in `im2col_w` mode using
-the `override.addr` operands described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
-The last boolean flag argument has the same functionality as described in the `s2g.tile` override intrinsics above.
+They copy tensor data from shared::cta to global memory in `im2col_w` mode
+using the `override.addr` operands described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
+The last boolean flag argument has the same functionality as described in the
+`s2g.tile` override intrinsics above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
@@ -2771,7 +2767,8 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.5d(...,
The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.override.addr.[3-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.im2col_no_offs.[3-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `im2col` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
-The `override.addr` operands are described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
+The `override.addr` operands are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
`%red_op` and the last boolean flag are as described for `reduce.tile` above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
@@ -2790,7 +2787,8 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.5d(..
The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.w.override.addr.[3-5]d`' intrinsics correspond to the `cp.reduce.async.bulk.tensor.im2col_no_offs::w.[3-5]d.global.shared::cta.*` set of PTX instructions qualified with the `.override::global_address` qualifier.
These instructions initiate an asynchronous reduction operation of tensor data in global memory with the tensor data in shared::cta memory, using `im2col_w` mode, while overriding the global base address from the tensor-map with the explicit `ptr addrspace(1) %override_addr` operand.
-The `override.addr` operands are described in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
+The `override.addr` operands are described in
+[Tensor-Map Property Overrides](#tensor-map-property-overrides).
`%red_op` and the last boolean flag are as described for `reduce.tile` above.
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index f9db6a8455230..e0c3283ca8723 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1312,6 +1312,25 @@ class TMA_IM2COL_OFFSETS_UTIL<int dim, string mode> {
true : 0);
}
+// Tile::gather4/scatter4 operate on a 2D tensor (5 co-ordinates), so they
+// always use a "2d" suffix regardless of dim.
+class TMA_DIMS_UTIL<int dim, string mode> {
+ int num_dims = !cond(
+ !eq(mode, "tile_gather4") : 2,
+ !eq(mode, "tile_scatter4") : 2,
+ true : dim);
+}
+
+// Modes applicable for a given tensor dimensionality. The im2col modes are
+// only defined for 3D and higher tensors; for dim < 3 only "tile" is
+// supported. `includeW128` selects whether "im2col_w_128" mode is included
+// in the mode list.
+class TMA_MODES_UTIL<int dim, bit includeW128 = 0> {
+ list<string> all_modes = !listconcat(["tile", "im2col", "im2col_w"],
+ !if(includeW128, ["im2col_w_128"], []));
+ list<string> modes = !if(!ge(dim, 3), all_modes, ["tile"]);
+}
+
// Class to handle TMA Copy Override Support
class TMAOverrideHandler<int dim, string mode> {
// TMA Override Support:
@@ -1355,10 +1374,11 @@ class CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<int dim, string mode, string override>
string Suffix = "_" # mode # "_" # override # "_" # dim # "d";
}
-// Helper that emits the intrinsic-property set for the cache_hint flag argument.
-class NVVM_CACHE_HINT_ARGPROP<int argNo> {
+// Helper that emits an ArgInfo intrinsic-property providing a name for the
+// given flag/immarg argument.
+class NVVM_ARG_NAME_PROP<int argNo, string name> {
IntrinsicProperty Prop =
- ArgInfo<ArgIndex<argNo>, [ArgName<"flag_cache_hint">]>;
+ ArgInfo<ArgIndex<argNo>, [ArgName<name>]>;
}
class CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode, string override>
@@ -1377,7 +1397,7 @@ class CP_ASYNC_BULK_TENSOR_S2G_OVERRIDE_INTR<int dim, string mode, string overri
list<IntrinsicProperty> IntrProp = [
IntrConvergent,
ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>, WriteOnly<ArgIndex<2>>,
- NVVM_CACHE_HINT_ARGPROP<FlagCacheHintIdx>.Prop
+ NVVM_ARG_NAME_PROP<FlagCacheHintIdx, "flag_cache_hint">.Prop
];
}
@@ -1395,7 +1415,7 @@ class CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode, string ove
Range<ArgIndex<RedOpIdx>, 0, 8>,
ArgInfo<ArgIndex<RedOpIdx>, [ArgName<"red_op">,
ImmArgPrinter<"printTMAReductionOp">]>,
- NVVM_CACHE_HINT_ARGPROP<!add(RedOpIdx, 1)>.Prop
+ NVVM_ARG_NAME_PROP<!add(RedOpIdx, 1), "flag_cache_hint">.Prop
];
}
@@ -1403,7 +1423,7 @@ class CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode, string ove
// `upperBound` is the exclusive upper bound of the allowed policy value range.
// Prefetch accepts both EVICT_NORMAL and EVICT_LAST (default 2).
// Applypriority only accepts EVICT_NORMAL (upperBound = 1).
-class NVVM_EVICT_POLICY_ARGPROP<int argNo, int upperBound = 2> {
+class NVVM_EVICT_POLICY_RANGED_ARGPROP<int argNo, int upperBound = 2> {
list<IntrinsicProperty> Prop = [
ImmArg<ArgIndex<argNo>>,
Range<ArgIndex<argNo>, 0, upperBound>,
@@ -1431,47 +1451,48 @@ class CP_ASYNC_BULK_TENSOR_PREFETCH_OVERRIDE_INTR<int dim, string mode,
list<IntrinsicProperty> IntrProp = !listconcat(
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>],
!if(IsEvict,
- NVVM_EVICT_POLICY_ARGPROP<PolicyIdx>.Prop,
- [NVVM_CACHE_HINT_ARGPROP<!size(ArgsTy)>.Prop]));
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<PolicyIdx>.Prop,
+ [NVVM_ARG_NAME_PROP<!size(ArgsTy), "flag_cache_hint">.Prop]));
}
// Applypriority TMA variants.
class APPLYPRIORITY_BULK_TENSOR_INTR<int dim, string mode> {
+ defvar num_dims = TMA_DIMS_UTIL<dim, mode>.num_dims;
string Name = "int_nvvm_applypriority_async_bulk_tensor_evict_priority_" #
- mode # "_" # !if(!eq(mode, "tile_gather4"), 2, dim) # "d";
+ mode # "_" # num_dims # "d";
int NumIm2ColOffsets = TMA_IM2COL_OFFSETS_UTIL<dim, mode>.num;
list<LLVMType> ArgsTy = !listconcat(
[llvm_ptr_ty], // tensormap_ptr
!listsplat(llvm_i32_ty, dim), // actual tensor dims
!listsplat(llvm_i16_ty, NumIm2ColOffsets), // im2col offsets
- [llvm_i32_ty]); // evict_policy
+ [llvm_i32_ty]); // flag_evict_policy
defvar PolicyIdx = !sub(!size(ArgsTy), 1);
list<IntrinsicProperty> IntrProp = !listconcat(
[IntrConvergent, ReadOnly<ArgIndex<0>>],
- NVVM_EVICT_POLICY_ARGPROP<PolicyIdx, 1>.Prop);
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<PolicyIdx, 1>.Prop);
}
// Applypriority TMA variants with override global address.
class APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<int dim, string mode,
string override>
: CP_ASYNC_BULK_TENSOR_OVERRIDE_BASE<dim, mode, override> {
+ defvar num_dims = TMA_DIMS_UTIL<dim, mode>.num_dims;
string Name = "int_nvvm_applypriority_async_bulk_tensor_evict_priority_" #
- mode # "_" # override # "_" #
- !if(!eq(mode, "tile_gather4"), 2, dim) # "d";
+ mode # "_" # override # "_" # num_dims # "d";
list<LLVMType> ArgsTy = !listconcat(
[llvm_ptr_ty], // tensormap_ptr
OverrideTypes, // override_addr (src)
TensorDimsTy, // actual tensor dims
Im2ColOffsetsTy, // im2col offsets
- [llvm_i32_ty]); // evict_policy
+ [llvm_i32_ty]); // flag_evict_policy
defvar PolicyIdx = !sub(!size(ArgsTy), 1);
list<IntrinsicProperty> IntrProp = !listconcat(
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>],
- NVVM_EVICT_POLICY_ARGPROP<PolicyIdx, 1>.Prop);
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<PolicyIdx, 1>.Prop);
}
let TargetPrefix = "nvvm" in {
@@ -3041,7 +3062,7 @@ class DefaultAttrsIntrinsicFlags<list<LLVMType> ret_types,
// TMA Tensor Copy Intrinsics: S2G -> From Shared to Global memory variants
foreach dim = 1...5 in {
defvar tensor_dim_args = !listsplat(llvm_i32_ty, dim);
- foreach mode = !if(!ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim>.modes in {
def int_nvvm_cp_async_bulk_tensor_s2g_ # mode # _ # dim # d :
DefaultAttrsIntrinsicFlags<[],
!listconcat([llvm_shared_ptr_ty, // src_smem_ptr
@@ -3083,7 +3104,7 @@ def int_nvvm_cp_async_bulk_tensor_s2g_tile_scatter4_2d
// TMA Tensor Copy/Reduction Intrinsics: S2G -> From shared to global memory
// with override address variants
foreach dim = 1...5 in {
- foreach mode = !if(!ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim>.modes in {
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
// Reduce + Override
foreach reduce_override =
@@ -3114,13 +3135,13 @@ def int_nvvm_cp_async_bulk_tensor_s2g_tile_scatter4_override_addr_2d
[llvm_i64_ty]), // cache_hint
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
- WriteOnly<ArgIndex<2>>, NVVM_CACHE_HINT_ARGPROP<9>.Prop]>;
+ WriteOnly<ArgIndex<2>>, NVVM_ARG_NAME_PROP<9, "flag_cache_hint">.Prop]>;
// TMA Tensor Copy Intrinsics: G2S -> From Global to Shared memory variants
foreach dim = 1...5 in {
defvar tensor_dim_args = !listsplat(llvm_i32_ty, dim);
- foreach mode = !if(!ge(dim, 3), ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim, /*includeW128=*/1>.modes in {
defvar is_im2col = !eq(mode, "im2col");
defvar is_im2colw = !or(!eq(mode, "im2col_w"), !eq(mode, "im2col_w_128"));
@@ -3164,9 +3185,9 @@ foreach dim = 1...5 in {
!listconcat([llvm_ptr_ty], // tensormap_ptr
tensor_dim_args, // actual tensor dims
im2col_offsets_args, // im2col offsets
- [llvm_i32_ty]), // evict_policy
+ [llvm_i32_ty]), // flag_evict_policy
!listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>],
- NVVM_EVICT_POLICY_ARGPROP<evict_policy_idx>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<evict_policy_idx>.Prop)>;
// Prefetch + Override
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
@@ -3242,9 +3263,9 @@ def int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_tile_gather4_2d
: DefaultAttrsIntrinsic<[],
!listconcat([llvm_ptr_ty], // tensormap_ptr
!listsplat(llvm_i32_ty, 5), // co-ordinates
- [llvm_i32_ty]), // evict_policy
+ [llvm_i32_ty]), // flag_evict_policy
!listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>],
- NVVM_EVICT_POLICY_ARGPROP<6>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<6>.Prop)>;
// TMA prefetch for tile::gather4 with override address
def int_nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_override_addr_2d
@@ -3255,7 +3276,7 @@ def int_nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_override_addr_2d
[llvm_i64_ty]), // cache_hint
[llvm_i1_ty], // Flag for cache_hint
[IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
- NVVM_CACHE_HINT_ARGPROP<8>.Prop]>;
+ NVVM_ARG_NAME_PROP<8, "flag_cache_hint">.Prop]>;
// TMA prefetch for tile::gather4 with override address and eviction priority
def int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_tile_gather4_override_addr_2d
@@ -3263,9 +3284,9 @@ def int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_tile_gather4_override_
!listconcat([llvm_ptr_ty, // tensormap_ptr
llvm_global_ptr_ty], // override_addr (src)
!listsplat(llvm_i32_ty, 5), // co-ordinates
- [llvm_i32_ty]), // evict_policy
+ [llvm_i32_ty]), // flag_evict_policy
!listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>],
- NVVM_EVICT_POLICY_ARGPROP<7>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<7>.Prop)>;
// Intrinsics for Prefetch and Prefetchu
let IntrProperties = [IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>] in {
@@ -3298,19 +3319,18 @@ def int_nvvm_applypriority_async_bulk_evict_priority
: DefaultAttrsIntrinsic<[],
[llvm_global_ptr_ty, // global_ptr
llvm_i32_ty, // size (byte length; must be a multiple of 16)
- llvm_i32_ty], // evict_policy
+ llvm_i32_ty], // flag_evict_policy
!listconcat([IntrConvergent, IntrArgMemOnly,
ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
- NVVM_EVICT_POLICY_ARGPROP<2, 1>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<2, 1>.Prop)>;
// Applypriority TMA variants.
foreach dim = 1...5 in {
- foreach mode = !if(!ge(dim, 3),
- ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
- foreach ap_tensor =
+ foreach mode = TMA_MODES_UTIL<dim, /*includeW128=*/1>.modes in {
+ foreach apply_prio =
[APPLYPRIORITY_BULK_TENSOR_INTR<dim, mode>] in
- def ap_tensor.Name : DefaultAttrsIntrinsic<[], ap_tensor.ArgsTy,
- ap_tensor.IntrProp>;
+ def apply_prio.Name : DefaultAttrsIntrinsic<[], apply_prio.ArgsTy,
+ apply_prio.IntrProp>;
}
}
// Applypriority TMA variants for tile::gather4.
@@ -3318,20 +3338,19 @@ def int_nvvm_applypriority_async_bulk_tensor_evict_priority_tile_gather4_2d
: DefaultAttrsIntrinsic<[],
!listconcat([llvm_ptr_ty], // tensormap_ptr
!listsplat(llvm_i32_ty, 5), // co-ordinates (d0..d4)
- [llvm_i32_ty]), // evict_policy
+ [llvm_i32_ty]), // flag_evict_policy
!listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>],
- NVVM_EVICT_POLICY_ARGPROP<6, 1>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<6, 1>.Prop)>;
// Applypriority TMA variants with override global address.
foreach dim = 1...5 in {
- foreach mode = !if(!ge(dim, 3),
- ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim, /*includeW128=*/1>.modes in {
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
- foreach ap_tensor_override =
+ foreach apply_prio_override =
[APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<dim, mode, override>] in
- def ap_tensor_override.Name
- : DefaultAttrsIntrinsic<[], ap_tensor_override.ArgsTy,
- ap_tensor_override.IntrProp>;
+ def apply_prio_override.Name
+ : DefaultAttrsIntrinsic<[], apply_prio_override.ArgsTy,
+ apply_prio_override.IntrProp>;
}
}
}
@@ -3341,10 +3360,10 @@ def int_nvvm_applypriority_async_bulk_tensor_evict_priority_tile_gather4_overrid
!listconcat([llvm_ptr_ty], // tensormap_ptr
[llvm_global_ptr_ty], // override_addr (src)
!listsplat(llvm_i32_ty, 5), // co-ordinates (d0..d4)
- [llvm_i32_ty]), // evict_policy
+ [llvm_i32_ty]), // flag_evict_policy
!listconcat([IntrConvergent, ReadOnly<ArgIndex<0>>,
ReadOnly<ArgIndex<1>>],
- NVVM_EVICT_POLICY_ARGPROP<7, 1>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<7, 1>.Prop)>;
// discard
let IntrProperties = [NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<1>>, IntrHasSideEffects] in {
@@ -3428,10 +3447,10 @@ def int_nvvm_cp_async_bulk_prefetch_evict_priority
: DefaultAttrsIntrinsic<[],
[llvm_global_ptr_ty, // src_gmem_ptr
llvm_i32_ty, // copy_size
- llvm_i32_ty], // evict_policy
+ llvm_i32_ty], // flag_evict_policy
!listconcat([IntrConvergent, IntrArgMemOnly,
NoCapture<ArgIndex<0>>, ReadOnly<ArgIndex<0>>],
- NVVM_EVICT_POLICY_ARGPROP<2>.Prop)>;
+ NVVM_EVICT_POLICY_RANGED_ARGPROP<2>.Prop)>;
def int_nvvm_griddepcontrol_launch_dependents : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
def int_nvvm_griddepcontrol_wait : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 142a0761d41b1..907c6cbb178fd 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -640,23 +640,12 @@ def APPLYPRIORITY_ASYNC_BULK_GLOBAL_EVICT_PRIORITY
// TMA Async Bulk Tensor Copy Functions
//-------------------------------------
-class TMA_DIMS_UTIL<int dim, string mode = ""> {
+class TMA_DIMS_DAG_UTIL<int dim> {
// For example, when 'dim' is 3, this generates:
// an ins_dag: B32:$d0, B32:$d1, B32:$d2
// with base_str: $d0, $d1, $d2
dag ins_dag = !dag(ins, !listsplat(B32, dim), !foreach(i, !range(dim), "d" # i));
string base_str = !interleave(!foreach(i, !range(dim), "$d" # i), ", ");
-
- // Tile::Gather4/scatter4 actually operate on a 2D tensor,
- // though they take 5 co-ordinates.
- //
- // The scatter-gather happens over 4 rows with a fixed
- // column-index. The first co-ordinate represents the
- // col-index followed by four row-indices.
- int num_dims = !cond(
- !eq(mode, "tile_scatter4") : 2,
- !eq(mode, "tile_gather4") : 2,
- true : dim); // for all other modes
}
class TMA_IM2COL_UTIL<int dim, string mode> {
@@ -690,8 +679,8 @@ def tma_cta_group_imm_any : TImmLeaf<i32, [{return Imm >= 0;}]>;
multiclass TMA_TENSOR_G2S_INTR<int dim, string mode, list<Predicate> pred,
TImmLeaf cta_group_type = tma_cta_group_imm_any> {
- defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
- defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_DAG_UTIL<dim>.base_str;
defvar asm_str_base = "$cg [$dst], [$tmap, {{" # dims_str # "}}], [$mbar]";
defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;
@@ -767,8 +756,8 @@ defm TMA_G2S_TILE_GATHER4_2D : TMA_TENSOR_G2S_INTR<5, "tile_gather4",
[hasTMABlackwellSupport]>;
multiclass TMA_TENSOR_G2S_CTA_INTR<int dim, string mode, list<Predicate> pred = []> {
- defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
- defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_DAG_UTIL<dim>.base_str;
defvar asm_str_base = " [$dst], [$tmap, {{" # dims_str # "}}], [$mbar]";
defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;
@@ -830,8 +819,8 @@ defm TMA_G2S_CTA_TILE_GATHER4_2D : TMA_TENSOR_G2S_CTA_INTR<5, "tile_gather4",
multiclass TMA_TENSOR_S2G_INTR<int dim, string mode,
list<Predicate> pred = [PTX80, SM90]> {
- defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
- defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_DAG_UTIL<dim>.base_str;
defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
defvar dim_val = TMA_DIMS_UTIL<dim, mode>.num_dims;
@@ -881,7 +870,7 @@ defm TMA_S2G_TILE_SCATTER4_2D : TMA_TENSOR_S2G_INTR<5, "tile_scatter4",
// Base class for TMA tensor override intrinsics
class TMA_TENSOR_OVERRIDE_BASE<int dim, string mode, string override> {
// Dimension handling.
- defvar dims_util = TMA_DIMS_UTIL<dim>;
+ defvar dims_util = TMA_DIMS_DAG_UTIL<dim>;
dag dims_dag = dims_util.ins_dag;
string dims_str = dims_util.base_str;
@@ -1032,8 +1021,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_REDUCE_OVERRIDE_INTR<int dim, string mode,
}
foreach dim = 1...5 in {
- foreach mode = !if(
- !ge(dim, 3), ["tile", "im2col", "im2col_w"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim>.modes in {
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
defvar suffix = dim # "D_" # !toupper(mode # "_" # override);
defm S2G_ # suffix
@@ -1050,8 +1038,8 @@ defm S2G_2D_TILE_SCATTER4_OVERRIDE_ADDR
// TMA Copy from Shared to Global memory with Reduction
multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, string mode,
list<Predicate> pred = [PTX80, SM90]> {
- defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
- defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_DAG_UTIL<dim>.base_str;
defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
defvar mode_asm_str = TMA_S2G_MODE_ASM_UTIL<mode>.ret;
@@ -1108,8 +1096,8 @@ foreach dim = 3...5 in {
// TMA Prefetch from Global memory to L2 cache
multiclass TMA_TENSOR_PREFETCH_INTR<int dim, string mode,
list<Predicate> pred = [PTX80, SM90]> {
- defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
- defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_DAG_UTIL<dim>.base_str;
defvar asm_str_base = " [$tmap, {{" # dims_str # "}}]";
defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;
@@ -1124,8 +1112,9 @@ multiclass TMA_TENSOR_PREFETCH_INTR<int dim, string mode,
# "." # "L2.global"
# "." # !subst("_", "::", mode);
+ defvar intr_suffix = mode # "_" # dim_val # "d";
defvar intr = !cast<Intrinsic>(
- "int_nvvm_cp_async_bulk_tensor_prefetch_" # mode # "_" # dim_val # "d");
+ "int_nvvm_cp_async_bulk_tensor_prefetch_" # intr_suffix);
defvar ins_dag = !con((ins B64:$tmap),
dims_dag,
@@ -1151,7 +1140,7 @@ multiclass TMA_TENSOR_PREFETCH_INTR<int dim, string mode,
defvar intr_evict =
!cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_" #
- mode # "_" # dim_val # "d");
+ intr_suffix);
defvar ins_dag_evict = !con((ins B64:$tmap), dims_dag, im2col_dag,
(ins EvictPolicyOp:$policy));
defvar intr_dag_evict = !con((intr_evict B64:$tmap),
@@ -1191,12 +1180,12 @@ multiclass NVPTX_TMA_TENSOR_PREFETCH_OVERRIDE_INTR<int dim, string mode,
defvar asm_str = " [$tmap, $override_addr, " # base.tensor_size_component #
"{{" # base.dims_str # "}}]" # base.im2col_component;
+ defvar intr_suffix = mode # "_" # override # "_" # num_dims # "d";
defvar intr =
- !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_" #
- mode # "_" # override # "_" # num_dims # "d");
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_" # intr_suffix);
defvar intr_evict =
!cast<Intrinsic>("int_nvvm_cp_async_bulk_tensor_prefetch_evict_priority_" #
- mode # "_" # override # "_" # num_dims # "d");
+ intr_suffix);
defvar ins_dag_base =
!con((ins B64:$tmap, B64:$override_addr),
@@ -1243,8 +1232,7 @@ multiclass NVPTX_TMA_TENSOR_PREFETCH_OVERRIDE_INTR<int dim, string mode,
}
foreach dim = 1...5 in {
- foreach mode = !if(!ge(dim, 3),
- ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim, /*includeW128=*/1>.modes in {
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
defvar suffix = !toupper(mode # "_" # override) # "_" # dim # "D";
defm TMA_TENSOR_PFO_ # suffix
@@ -1257,8 +1245,8 @@ defm TMA_TENSOR_PFO_TILE_GATHER4_OVERRIDE_ADDR
// TMA applypriority variants.
multiclass NVPTX_APPLYPRIORITY_BULK_TENSOR_INTR<int dim, string mode> {
- defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
- defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
+ defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
+ defvar dims_str = TMA_DIMS_DAG_UTIL<dim>.base_str;
defvar asm_str_base = " [$tmap, {{" # dims_str # "}}]";
defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;
@@ -1291,8 +1279,7 @@ multiclass NVPTX_APPLYPRIORITY_BULK_TENSOR_INTR<int dim, string mode> {
}
foreach dim = 1...5 in {
- foreach mode = !if(!ge(dim, 3),
- ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim, /*includeW128=*/1>.modes in {
defvar suffix = !toupper(mode) # "_" # dim # "D";
defm NVPTX_APPLYPRIORITY_BULK_TENSOR_ # suffix
: NVPTX_APPLYPRIORITY_BULK_TENSOR_INTR<dim, mode>;
@@ -1339,8 +1326,7 @@ multiclass NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_INTR<int dim, string mode,
}
foreach dim = 1...5 in {
- foreach mode = !if(!ge(dim, 3),
- ["tile", "im2col", "im2col_w", "im2col_w_128"], ["tile"]) in {
+ foreach mode = TMA_MODES_UTIL<dim, /*includeW128=*/1>.modes in {
foreach override = TMAOverrideHandler<dim, mode>.applicable_overrides in {
defvar suffix = !toupper(mode # "_" # override) # "_" # dim # "D";
defm NVPTX_APPLYPRIORITY_BULK_TENSOR_OVERRIDE_ # suffix
More information about the llvm-commits
mailing list