[Mlir-commits] [mlir] 83066e0 - [LLVM][NVPTX] Add async bulk copy global to shared extensions (#222323)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Tue Sep 15 22:16:18 PDT 2026
Author: Rajat Bajpai
Date: 2026-09-16T10:46:13+05:30
New Revision: 83066e0d57798b10c759fea6ab77346dabc0e1dc
URL: https://github.com/llvm/llvm-project/commit/83066e0d57798b10c759fea6ab77346dabc0e1dc
DIFF: https://github.com/llvm/llvm-project/commit/83066e0d57798b10c759fea6ab77346dabc0e1dc.diff
LOG: [LLVM][NVPTX] Add async bulk copy global to shared extensions (#222323)
This change adds following things to bulk copy intrinsics.
1. Relaxed memory ordering semantics with a scope argument.
2. Data-validity reporting patterns (introduced in Rubin).
3. 32-bit multicast mask for global to shared::cluster variants
(introduced in Rubin).
4. Ignore out of bound checks for global to shared::cta variants.
Note: MLIR lowering is updated to emit the new intrinsic signatures.
Support for the new features in MLIR will be done in a separate change.
Added:
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-invalid.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-oob.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-relaxed.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-validate-data.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-invalid.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-mc32.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-relaxed.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-validate-data.ll
llvm/test/Verifier/NVPTX/cp-async-bulk-g2s-cta.ll
Modified:
llvm/docs/NVPTXUsage.md
llvm/include/llvm/IR/IntrinsicsNVVM.td
llvm/include/llvm/IR/NVVMIntrinsicUtils.h
llvm/lib/IR/AutoUpgrade.cpp
llvm/lib/IR/NVVMIntrinsicUtils.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-override.ll
llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-validate-data.ll
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
mlir/test/Target/LLVMIR/nvvm/tma_bulk_copy.mlir
mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_im2col.mlir
mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_tile.mlir
mlir/test/Target/LLVMIR/nvvm/tma_load_cta_im2col.mlir
mlir/test/Target/LLVMIR/nvvm/tma_load_cta_tile.mlir
Removed:
################################################################################
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index ee12c940a5e72..67bf8ea482933 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1761,8 +1761,7 @@ For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parall
#### TMA Global-to-Shared Data-Validity Reporting
The '`@llvm.nvvm.cp.async.bulk.tensor.g2s.*`' and '`@llvm.nvvm.cp.async.bulk.tensor.g2s.cta.*`' intrinsics, including the property-override
-forms, take an `i32 %validate_pattern` immediate argument in the range \[0, 6). It selects the data-validity reporting mechanism applied to the
-tensor data while it is copied:
+forms, take an `i32 %flag_valid_pattern` immediate argument in the range \[0, 6). The non-tensor '`@llvm.nvvm.cp.async.bulk.global.to.shared.cluster`', '`@llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed`', '`@llvm.nvvm.cp.async.bulk.global.to.shared.cta`', and '`@llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed`' intrinsics also take the same `i32 %flag_valid_pattern` immediate argument. It selects the data-validity reporting mechanism applied to the data while it is copied:
| Value | PTX Report Mechanism | Semantics |
|-------|---------------------------------------------------------|------------------------------------------------------------------|
@@ -1784,7 +1783,8 @@ For more information, refer to the [PTX ISA](https://docs.nvidia.com/cuda/parall
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %flag_valid_pattern)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i32(..., i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %flag_valid_pattern)
```
##### Overview:
@@ -1795,16 +1795,71 @@ instructions. These instructions initiate an asynchronous copy of bulk data from
global memory to shared::cluster memory. The 32-bit operand `%size` specifies
the amount of memory to be copied and it must be a multiple of 16.
-- The last two arguments to these intrinsics are boolean flags indicating
- support for cache_hint and/or multicast modifiers. These flag arguments must
- be compile-time constants. The backend looks through these flags and lowers
- the intrinsics appropriately.
-- The Nth argument (denoted by `i1 %flag_ch`) when set, indicates a valid
- cache_hint (`i64 %ch`) and generates the `.L2::cache_hint` variant of the
- PTX instruction.
-- The [N-1]th argument (denoted by `i1 %flag_mc`) when set, indicates the
- presence of a multicast mask (`i16 %mc`) and generates the PTX instruction
- with the `.multicast::cluster` modifier.
+- The trailing `%flag_mc`, `%flag_ch`, and `%flag_valid_pattern` arguments control
+ the multicast and cache-hint modifiers and the data-validity reporting
+ pattern. They must be compile-time constants.
+- The argument denoted by `i1 %flag_ch`, when set, indicates a valid cache_hint
+ (`i64 %ch`) and generates the `.L2::cache_hint` variant of the PTX
+ instruction.
+- The argument denoted by `i1 %flag_mc`, when set, indicates the presence of a
+ multicast mask (`%mc`) and generates the PTX instruction with the
+ `.multicast::cluster` modifier. An `i16` mask selects the default/16-bit
+ multicast form while an `i32` mask selects the `.multicast::cluster::32b`
+ variant. Only `i16` and `i32` mask types are supported. The 32-bit form
+ requires `sm_107f` or later in the same family and is supported only on PTX
+ 9.4 or later.
+- The trailing `i32 %flag_valid_pattern` flag selects the data-validity reporting
+ mechanism; see
+ [TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk).
+
+#### '`llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %flag_scope, i32 %flag_valid_pattern)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i32(..., i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %flag_scope, i32 %flag_valid_pattern)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed`' intrinsic
+corresponds to the
+`cp.async.bulk.relaxed.<scope>.shared::cluster.global.mbarrier::complete_tx::bytes{...}.b128`
+family of PTX instructions. These instructions initiate an asynchronous copy of bulk data from
+global memory to shared::cluster memory, like the weak variant above. Unlike
+the weak variant, the copy accesses memory with naturally-aligned strong
+memory operations with element-wise atomic size specified by the `.b128` type (implicit) and
+thread scope specified by `%flag_scope`; the complete-tx operation on the mbarrier has `.release` semantics at the
+`.cluster` scope.
+
+- The trailing `%flag_mc`, `%flag_ch`, `%flag_scope`, and `%flag_valid_pattern`
+ arguments control the multicast and cache-hint modifiers, the scope of the
+ relaxed memory ordering semantics, and the data-validity reporting pattern.
+ They must be compile-time constants.
+- The argument denoted by `i1 %flag_ch`, when set, indicates a valid cache_hint
+ (`i64 %ch`) and generates the `.L2::cache_hint` variant of the PTX
+ instruction.
+- The argument denoted by `i1 %flag_mc`, when set, indicates the presence of a
+ multicast mask (`%mc`) and generates the PTX instruction with the
+ `.multicast::cluster` modifier. An `i16` mask selects the default/16-bit
+ multicast form while an `i32` mask selects the `.multicast::cluster::32b`
+ variant. Only `i16` and `i32` mask types are supported.
+- The `i32 %flag_scope` argument takes values within the range \[0, 4) and selects
+ the `.scope` qualifier of the PTX instruction:
+
+| Value | `flag_scope` |
+|-------|--------------|
+| 0 | `.cta` |
+| 1 | `.cluster` |
+| 2 | `.gpu` |
+| 3 | `.sys` |
+
+- The trailing `i32 %flag_valid_pattern` flag selects the data-validity reporting
+ mechanism; see
+ [TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk).
@@ -1813,7 +1868,7 @@ For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thre
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %mbar, ptr addrspace(1) %src, i32 %size, i64 %ch, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %mbar, ptr addrspace(1) %src, i32 %size, i32 %ibl, i32 %ibr, i64 %ch, i1 %flag_ch, i1 %flag_oob, i32 %flag_valid_pattern)
```
##### Overview:
@@ -1822,10 +1877,75 @@ The '`@llvm.nvvm.cp.async.bulk.global.to.shared.cta`' intrinsic corresponds to
the `cp.async.bulk.shared::cta.global.*` family of PTX instructions. These
instructions initiate an asynchronous copy of bulk data from global memory to
shared::cta memory. The 32-bit operand `%size` specifies the amount of memory
-to be copied and it must be a multiple of 16. The last argument (denoted by
-`i1 %flag_ch`) is a compile-time constant. When set, it indicates a valid
-cache_hint (`i64 %ch`) and generates the `.L2::cache_hint` variant of the
-PTX instruction.
+to be copied and it must be a multiple of 16.
+
+- The trailing `%flag_ch`, `%flag_oob`, and `%flag_valid_pattern` arguments
+ control the cache-hint modifier, the out-of-bounds handling, and the
+ data-validity reporting pattern. They must be compile-time constants.
+- The argument denoted by `i1 %flag_ch`, when set, indicates a valid cache_hint
+ (`i64 %ch`) and generates the `.L2::cache_hint` variant of the PTX
+ instruction.
+- The argument denoted by `i1 %flag_oob`, when set, generates the
+ `.ignore_oob` variant of the PTX instruction. In that case, the `i32 %ibl`
+ and `i32 %ibr` operands specify the numbers of bytes to ignore on the left
+ and right sides of the source, respectively. Both operands must be in the
+ range \[0, 15]; otherwise, the behavior is undefined. The `.ignore_oob`
+ modifier requires PTX ISA 9.2 or later. When `%flag_oob` is set,
+ `%flag_valid_pattern` must be 0 (data-validity reporting disabled).
+- The trailing `i32 %flag_valid_pattern` flag selects the data-validity reporting
+ mechanism; see
+ [TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk).
+
+#### '`llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %mbar, ptr addrspace(1) %src, i32 %size, i32 %ibl, i32 %ibr, i64 %ch, i1 %flag_ch, i1 %flag_oob, i32 %flag_scope, i32 %flag_valid_pattern)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed`' intrinsic
+corresponds to the
+`cp.async.bulk.relaxed.<scope>.shared::cta.global.mbarrier::complete_tx::bytes{...}.b128`
+family of PTX instructions. These instructions initiate an asynchronous copy of bulk data from
+global memory to shared::cta memory, like the weak variant above. Unlike
+the weak variant, the copy accesses memory with naturally-aligned strong
+memory operations with element-wise atomic size specified by the `.b128` type (implicit) and
+thread scope specified by `%flag_scope`; the complete-tx operation on the mbarrier has `.release` semantics at the
+`.cluster` scope. The `.relaxed` variants require PTX ISA 9.3 or later and
+`sm_90a`, `sm_100f`, or `sm_110f` or later in the same family.
+
+- The trailing `%flag_ch`, `%flag_oob`, `%flag_scope`, and `%flag_valid_pattern`
+ arguments control the cache-hint modifier, the out-of-bounds handling, the
+ scope of the relaxed memory ordering semantics, and the data-validity
+ reporting pattern. They must be compile-time constants.
+- The argument denoted by `i1 %flag_ch`, when set, indicates a valid cache_hint
+ (`i64 %ch`) and generates the `.L2::cache_hint` variant of the PTX
+ instruction.
+- The argument denoted by `i1 %flag_oob`, when set, generates the
+ `.ignore_oob` variant of the PTX instruction. In that case, the `i32 %ibl`
+ and `i32 %ibr` operands specify the numbers of bytes to ignore on the left
+ and right sides of the source, respectively. Both operands must be in the
+ range \[0, 15]; otherwise, the behavior is undefined. When `%flag_oob` is
+ set, `%flag_valid_pattern` must be 0 (data-validity reporting disabled); this
+ mutual exclusion is enforced by the verifier.
+- The `i32 %flag_scope` argument takes values within the range \[0, 4) and selects
+ the `.scope` qualifier of the PTX instruction:
+
+| Value | `flag_scope` |
+|-------|--------------|
+| 0 | `.cta` |
+| 1 | `.cluster` |
+| 2 | `.gpu` |
+| 3 | `.sys` |
+
+- The trailing `i32 %flag_valid_pattern` flag selects the data-validity reporting
+ mechanism; see
+ [TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk).
@@ -2334,14 +2454,14 @@ and to the [PTX ISA discard documentation](https://docs.nvidia.com/cuda/parallel
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i32(..., i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i32(..., i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.{i16,i32}(..., i32 %d0, i32 %d1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i32(..., i32 %mc, i64 %ch, i1 %flag_mc, ...)
```
@@ -2377,7 +2497,7 @@ takes a total of 5 coordinates as input arguments. For more information on
\[0, 3). The default value is '0' emits no cta_group modifier in the instruction.
The values '1' and '2' lower to `cta_group::1` and `cta_group::2` variants of the PTX instruction
respectively.
-- The last `i32 %validate_pattern` flag selects the data-validity reporting
+- The last `i32 %flag_valid_pattern` flag selects the data-validity reporting
mechanism; see
[TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
@@ -2388,17 +2508,17 @@ For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thre
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i32(..., i16 %im2col0, i32 %mc, i64 %ch, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i32(..., i16 %im2col0, i32 %mc, i64 %ch, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i32(..., i16 %im2col0, i32 %mc, i64 %ch, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
@@ -2430,13 +2550,13 @@ For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thre
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.2d(..., i32 %d0, i32 %d1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
```
##### Overview:
@@ -2458,7 +2578,7 @@ takes a total of 5 coordinates as input arguments. For more information on
reporting pattern. They must be compile-time constants. When `i1 %flag_ch`
is set, it indicates a valid cache hint (`i64 %ch`) and generates the
`.L2::cache_hint` variant of the PTX instruction.
-- The last `i32 %validate_pattern` flag selects the data-validity reporting
+- The last `i32 %flag_valid_pattern` flag selects the data-validity reporting
mechanism; see
[TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
@@ -2469,15 +2589,15 @@ For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thre
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
```
@@ -2508,19 +2628,19 @@ For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thre
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.1d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.1d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.1d.i32(..., i32 %mc, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.2d.{i16,i32}(..., i32 %d0, i32 %d1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.3d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.override.addr.2d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.override.addr.2d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.override.addr.2d.i32(..., i32 %mc, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.1d.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.1d.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.stride.2d.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.stride.2d.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i32 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.stride.3d.i32(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.stride.4d.i32(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.dim.stride.5d.i32(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
@@ -2541,7 +2661,7 @@ in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
| 0 | no cta group modifier (default) |
| 1 | `cta_group::1` |
| 2 | `cta_group::2` |
-- The last `i32 %validate_pattern` flag is described in
+- The last `i32 %flag_valid_pattern` flag is described in
[TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
All above described flag operands must be compile time constants.
@@ -2555,17 +2675,17 @@ with `.override::global_dim` or `.override::global_dim_stride`, and require
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.override.addr.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.override.addr.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.override.addr.3d.i32(..., i32 %mc, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.override.addr.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.override.addr.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.override.addr.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.override.addr.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.override.addr.3d.i32(..., i32 %mc, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.override.addr.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.override.addr.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.override.addr.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.override.addr.3d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.override.addr.3d.i32(..., i32 %mc, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.override.addr.4d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.override.addr.5d.{i16,i32}(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
@@ -2578,7 +2698,7 @@ while allowing overriding properties in the opaque tensor map with
explicit operands. The override operands and their restrictions are described
in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
-The `%flag_mc`, `%flag_ch`, `%cta_group`, and `%validate_pattern` operands
+The `%flag_mc`, `%flag_ch`, `%cta_group`, and `%flag_valid_pattern` operands
have the same functionality as described in the `tile` mode override
intrinsics above. All these flag operands must be compile-time constants.
@@ -2591,17 +2711,17 @@ with `.override::global_dim` or `.override::global_dim_stride`, and require
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.1d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.1d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.2d(..., i32 %d0, i32 %d1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.override.addr.2d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.override.addr.2d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %x0, i32 %y0, i32 %y1, i32 %y2, i32 %y3, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.1d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.1d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i32 %d0, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.stride.2d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.stride.2d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i16 %ts0, i16 %ts1, i32 %stride0, i16 %upper_stride, i32 %d0, i32 %d1, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.stride.3d(..., i16 %ts0, i16 %ts1, i16 %ts2, i32 %stride0, i32 %stride1, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.stride.4d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i32 %stride0, i32 %stride1, i32 %stride2, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.override.addr.dim.stride.5d(..., i16 %ts0, i16 %ts1, i16 %ts2, i16 %ts3, i16 %ts4, i32 %stride0, i32 %stride1, i32 %stride2, i32 %stride3, i16 %upper_stride, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
@@ -2615,7 +2735,7 @@ explicit operands. The override operands and their restrictions are described
in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
- `%flag_ch` controls whether `%ch` cache hint is emitted.
-- The last `i32 %validate_pattern` flag is described in
+- The last `i32 %flag_valid_pattern` flag is described in
[TMA Global-to-Shared Data-Validity Reporting](#tma-global-to-shared-data-validity-reporting).
All above described flag operands must be compile time constants.
@@ -2629,15 +2749,15 @@ with `.override::global_dim` or `.override::global_dim_stride`, and require
##### Syntax:
```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.override.addr.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.override.addr.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.override.addr.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.override.addr.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.override.addr.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %validate_pattern)
+declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.override.addr.3d(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr %tensor_map, ptr addrspace(1) %override_addr, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 %flag_ch, i32 %flag_valid_pattern)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.override.addr.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.override.addr.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
```
@@ -2649,7 +2769,7 @@ while allowing overriding properties in the opaque tensor map with
explicit operands. The override operands and their restrictions are described
in [Tensor-Map Property Overrides](#tensor-map-property-overrides).
-The `%flag_ch` and `%validate_pattern` operands have the same functionality
+The `%flag_ch` and `%flag_valid_pattern` operands have the same functionality
as described in the `tile` mode override intrinsics above. All these flag
operands must be compile-time constants.
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 24f5a871d68f9..9c4de2b903a9a 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1463,17 +1463,27 @@ class NVVM_EVICT_POLICY_RANGED_ARGPROP<int argNo, int upperBound = 2> {
];
}
-// Helper that emits the intrinsic-property set for the `validate_pattern`
+// Helper that emits the intrinsic-property set for the `flag_valid_pattern`
// immediate argument used by the TMA g2s (global-to-shared copy) intrinsics.
// The allowed values correspond to `TMAValidateDataPattern`, i.e. [0, 6).
class NVVM_TMA_VALIDATE_PATTERN_ARGPROP<int argNo> {
list<IntrinsicProperty> Prop = [
Range<ArgIndex<argNo>, 0, 6>,
- ArgInfo<ArgIndex<argNo>, [ArgName<"validate_pattern">,
+ ArgInfo<ArgIndex<argNo>, [ArgName<"flag_valid_pattern">,
ImmArgPrinter<"printTMAValidateDataPattern">]>
];
}
+// Helper that emits the intrinsic-property set for the `flag_scope` immediate
+// argument. The allowed values are: cta, cluster, gpu, sys. i.e. [0, 4).
+class NVVM_MEM_SCOPE_ARGPROP<int argNo> {
+ list<IntrinsicProperty> Prop = [
+ Range<ArgIndex<argNo>, 0, 4>,
+ ArgInfo<ArgIndex<argNo>, [ArgName<"flag_scope">,
+ ImmArgPrinter<"printMemScope">]>
+ ];
+}
+
class CP_ASYNC_BULK_TENSOR_PREFETCH_OVERRIDE_INTR<int dim, string mode,
string override,
bit IsEvict = 0>
@@ -1515,7 +1525,7 @@ class CP_ASYNC_BULK_TENSOR_G2S_OVERRIDE_INTR<int dim, string mode, string overri
list<LLVMType> FlagsTy = [llvm_i1_ty, // Flag for cta_mask
llvm_i1_ty, // Flag for cache_hint
llvm_i32_ty, // cta_group
- llvm_i32_ty]; // validate_pattern
+ llvm_i32_ty]; // flag_valid_pattern
defvar CtaGroupIdx = !add(!size(ArgsTy), 2);
defvar ValidateIdx = !add(!size(ArgsTy), 3);
list<IntrinsicProperty> IntrProp = !listconcat(
@@ -1537,7 +1547,7 @@ class CP_ASYNC_BULK_TENSOR_G2S_CTA_OVERRIDE_INTR<int dim, string mode, string ov
Im2ColOffsetsTy, // im2col offsets
[llvm_i64_ty]); // cache_hint
list<LLVMType> FlagsTy = [llvm_i1_ty, // Flag for cache_hint
- llvm_i32_ty]; // validate_pattern
+ llvm_i32_ty]; // flag_valid_pattern
defvar ValidateIdx = !add(!size(ArgsTy), 1);
list<IntrinsicProperty> IntrProp = !listconcat(
[IntrConvergent, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
@@ -3275,7 +3285,7 @@ foreach dim = 1...5 in {
defvar g2s_flags = [llvm_i1_ty, // Flag for cta_mask
llvm_i1_ty, // Flag for cache_hint
llvm_i32_ty, // cta_group
- llvm_i32_ty]; // validate_pattern
+ llvm_i32_ty]; // flag_valid_pattern
defvar cta_group_idx = !add(
!size(g2s_params),
!sub(!size(g2s_flags), 2));
@@ -3351,7 +3361,7 @@ foreach dim = 1...5 in {
im2col_offsets_args, // im2col offsets
[llvm_i64_ty]), // cache_hint
[llvm_i1_ty, // Flag for cache_hint
- llvm_i32_ty], // validate_pattern
+ llvm_i32_ty], // flag_valid_pattern
!listconcat([IntrConvergent, WriteOnly<ArgIndex<0>>,
ReadOnly<ArgIndex<2>>],
NVVM_TMA_VALIDATE_PATTERN_ARGPROP<g2s_cta_validate_idx>.Prop)>;
@@ -3371,7 +3381,7 @@ def int_nvvm_cp_async_bulk_tensor_g2s_tile_gather4_2d
[llvm_i1_ty, // Flag for cta_mask
llvm_i1_ty, // Flag for cache_hint
llvm_i32_ty, // cta_group
- llvm_i32_ty], // validate_pattern
+ llvm_i32_ty], // flag_valid_pattern
!listconcat([IntrConvergent,
WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
// Allowed values for cta_group are [0, 3).
@@ -3387,7 +3397,7 @@ def int_nvvm_cp_async_bulk_tensor_g2s_cta_tile_gather4_2d
!listsplat(llvm_i32_ty, 5), // co-ordinates
[llvm_i64_ty]), // cache_hint
[llvm_i1_ty, // Flag for cache_hint
- llvm_i32_ty], // validate_pattern
+ llvm_i32_ty], // flag_valid_pattern
!listconcat([IntrConvergent,
WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>],
NVVM_TMA_VALIDATE_PATTERN_ARGPROP<10>.Prop)>;
@@ -3406,7 +3416,7 @@ def int_nvvm_cp_async_bulk_tensor_g2s_tile_gather4_override_addr_2d
[llvm_i1_ty, // Flag for cta_mask
llvm_i1_ty, // Flag for cache_hint
llvm_i32_ty, // cta_group
- llvm_i32_ty], // validate_pattern
+ llvm_i32_ty], // flag_valid_pattern
!listconcat([IntrConvergent,
WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
ReadOnly<ArgIndex<3>>,
@@ -3425,7 +3435,7 @@ def int_nvvm_cp_async_bulk_tensor_g2s_cta_tile_gather4_override_addr_2d
!listsplat(llvm_i32_ty, 5), // co-ordinates
[llvm_i64_ty]), // cache_hint
[llvm_i1_ty, // Flag for cache_hint
- llvm_i32_ty], // validate_pattern
+ llvm_i32_ty], // flag_valid_pattern
!listconcat([IntrConvergent,
WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
ReadOnly<ArgIndex<3>>],
@@ -3554,31 +3564,75 @@ let IntrProperties = [NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<1>>, IntrHasSideEf
}
// Intrinsics for Bulk Copy using TMA (non-tensor)
-// From Global to Shared Cluster
+// From Global to Shared Cluster (weak memory ordering).
def int_nvvm_cp_async_bulk_global_to_shared_cluster
: DefaultAttrsIntrinsicFlags<[],
[llvm_shared_cluster_ptr_ty, // dst_shared_cluster_ptr
llvm_shared_ptr_ty, // mbarrier_ptr
llvm_global_ptr_ty, // src_gmem_ptr
llvm_i32_ty, // copy_size
- llvm_i16_ty, // cta_mask
+ llvm_anyint_ty, // cta_mask (i16 or i32 overload)
llvm_i64_ty], // cache_hint
[llvm_i1_ty, // Flag for cta_mask
- llvm_i1_ty], // Flag for cache_hint
- [IntrConvergent, IntrArgMemOnly,
- WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>]>;
+ llvm_i1_ty, // Flag for cache_hint
+ llvm_i32_ty], // flag_valid_pattern
+ !listconcat([IntrConvergent, IntrArgMemOnly,
+ WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>],
+ NVVM_TMA_VALIDATE_PATTERN_ARGPROP<8>.Prop)>;
+
+// From Global to Shared Cluster (relaxed memory ordering semantics).
+def int_nvvm_cp_async_bulk_global_to_shared_cluster_relaxed
+ : DefaultAttrsIntrinsicFlags<[],
+ [llvm_shared_cluster_ptr_ty, // dst_shared_cluster_ptr
+ llvm_shared_ptr_ty, // mbarrier_ptr
+ llvm_global_ptr_ty, // src_gmem_ptr
+ llvm_i32_ty, // copy_size
+ llvm_anyint_ty, // cta_mask (i16 or i32 overload)
+ llvm_i64_ty], // cache_hint
+ [llvm_i1_ty, // Flag for cta_mask
+ llvm_i1_ty, // Flag for cache_hint
+ llvm_i32_ty, // flag_scope (cta/cluster/gpu/sys)
+ llvm_i32_ty], // flag_valid_pattern
+ !listconcat([IntrConvergent, IntrArgMemOnly,
+ WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>],
+ NVVM_MEM_SCOPE_ARGPROP<8>.Prop,
+ NVVM_TMA_VALIDATE_PATTERN_ARGPROP<9>.Prop)>;
-// From Global to Shared CTA
+// From Global to Shared CTA (weak memory ordering).
def int_nvvm_cp_async_bulk_global_to_shared_cta
: DefaultAttrsIntrinsicFlags<[],
[llvm_shared_ptr_ty, // dst_shared_cta_ptr
llvm_shared_ptr_ty, // mbarrier_ptr
llvm_global_ptr_ty, // src_gmem_ptr
llvm_i32_ty, // copy_size
+ llvm_i32_ty, // ignore_bytes_left (controlled by flag_oob)
+ llvm_i32_ty, // ignore_bytes_right (controlled by flag_oob)
llvm_i64_ty], // cache_hint
- [llvm_i1_ty], // Flag for cache_hint
- [IntrConvergent, IntrArgMemOnly,
- WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>]>;
+ [llvm_i1_ty, // Flag for cache_hint
+ llvm_i1_ty, // Flag for ignore_oob
+ llvm_i32_ty], // flag_valid_pattern
+ !listconcat([IntrConvergent, IntrArgMemOnly,
+ WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>],
+ NVVM_TMA_VALIDATE_PATTERN_ARGPROP<9>.Prop)>;
+
+// From Global to Shared CTA (relaxed memory ordering semantics).
+def int_nvvm_cp_async_bulk_global_to_shared_cta_relaxed
+ : DefaultAttrsIntrinsicFlags<[],
+ [llvm_shared_ptr_ty, // dst_shared_cta_ptr
+ llvm_shared_ptr_ty, // mbarrier_ptr
+ llvm_global_ptr_ty, // src_gmem_ptr
+ llvm_i32_ty, // copy_size
+ llvm_i32_ty, // ignore_bytes_left (controlled by flag_oob)
+ llvm_i32_ty, // ignore_bytes_right (controlled by flag_oob)
+ llvm_i64_ty], // cache_hint
+ [llvm_i1_ty, // Flag for cache_hint
+ llvm_i1_ty, // Flag for ignore_oob
+ llvm_i32_ty, // flag_scope (cta/cluster/gpu/sys)
+ llvm_i32_ty], // flag_valid_pattern
+ !listconcat([IntrConvergent, IntrArgMemOnly,
+ WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>],
+ NVVM_MEM_SCOPE_ARGPROP<9>.Prop,
+ NVVM_TMA_VALIDATE_PATTERN_ARGPROP<10>.Prop)>;
// From Shared CTA to Shared Cluster
def int_nvvm_cp_async_bulk_shared_cta_to_cluster
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index c6f3567791944..9585cdac2a4ea 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -103,6 +103,28 @@ inline StringRef getTMAValidateDataPatternName(TMAValidateDataPattern Pattern) {
llvm_unreachable("invalid TMA validate data pattern");
}
+// Scope of the memory ordering semantics.
+enum class MemScope : uint8_t {
+ CTA = 0,
+ CLUSTER = 1,
+ GPU = 2,
+ SYS = 3,
+};
+
+inline StringRef getMemScopeName(MemScope Scope) {
+ switch (Scope) {
+ case MemScope::CTA:
+ return "cta";
+ case MemScope::CLUSTER:
+ return "cluster";
+ case MemScope::GPU:
+ return "gpu";
+ case MemScope::SYS:
+ return "sys";
+ }
+ llvm_unreachable("invalid memory scope");
+}
+
// Eviction priorities applicable for prefetch and applypriority intrinsics.
enum class EvictPolicyType : uint8_t {
EVICT_NORMAL = 0, // default
@@ -204,6 +226,8 @@ LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
LLVM_ABI void printTMAValidateDataPattern(raw_ostream &OS,
const Constant *ImmArgVal);
+LLVM_ABI void printMemScope(raw_ostream &OS, const Constant *ImmArgVal);
+
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
const Constant *ImmArgVal);
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 5cb3d4346e4fa..75fc03c22222b 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1193,7 +1193,7 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
//
-// The current tail appends a trailing i32 %validate_pattern, so both
+// The current tail appends a trailing i32 %flag_valid_pattern, so both
// legacy tails are recognized by an i1 at parameter N-2.
static Intrinsic::ID
shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
@@ -1215,7 +1215,7 @@ shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
size_t NumParams = F->getFunctionType()->getNumParams();
// Parameter N-2 is i1 for both legacy tails; the current tail ends
- // with i32 %cta_group, i32 %validate_pattern, for which N-2 is i32.
+ // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
return Intrinsic::not_intrinsic;
@@ -1237,7 +1237,7 @@ shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
//
// arg1, arg2, .. i64 %ch, i1 %flag_ch
//
-// The current tail appends a trailing i32 %validate_pattern, so the
+// The current tail appends a trailing i32 %flag_valid_pattern, so the
// legacy tail is recognized by an i1 at parameter N-1.
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
StringRef Name) {
@@ -1256,7 +1256,7 @@ static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
return ID;
// Parameter N-1 is i1 for the legacy tail; the current tail ends
- // with i32 %validate_pattern, for which N-1 is i32.
+ // with i32 %flag_valid_pattern, for which N-1 is i32.
if (!F->getFunctionType()
->getParamType(F->getFunctionType()->getNumParams() - 1)
->isIntegerTy(1))
@@ -1264,6 +1264,55 @@ static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
return ID;
}
+// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
+//
+// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
+//
+// The current intrinsic is overloaded on the multicast-mask type and takes a
+// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
+// parameter N-1.
+static Intrinsic::ID
+shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
+ SmallVectorImpl<Type *> &OvlTys) {
+ if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
+ return Intrinsic::not_intrinsic;
+
+ // Parameter N-1 is i1 for the legacy tail; the current tail ends with
+ // i32 %flag_valid_pattern, for which N-1 is i32.
+ size_t NumParams = F->getFunctionType()->getNumParams();
+ if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
+ return Intrinsic::not_intrinsic;
+
+ // The multicast mask is parameter 4; legacy IR only uses i16.
+ Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
+ if (!MaskTy->isIntegerTy(16))
+ return Intrinsic::not_intrinsic;
+ OvlTys.push_back(MaskTy);
+
+ return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
+}
+
+// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
+//
+// ..., i64 %ch, i1 %flag_ch
+//
+// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
+// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
+// recognized by an i1 at parameter N-1, whereas the current tail ends
+// with an i32.
+static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
+ StringRef Name) {
+ if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
+ return Intrinsic::not_intrinsic;
+
+ // Parameter N-1 is i1 for the legacy tail; the current tail ends with
+ // i32 %flag_valid_pattern, for which N-1 is i32.
+ if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
+ return Intrinsic::not_intrinsic;
+
+ return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
+}
+
// The legacy TMA reduction intrinsics encode the reduction operator in their
// name, while the current ones take it as an immediate argument. Map the
// operator part of a legacy name to the corresponding immediate value.
@@ -1310,8 +1359,6 @@ static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
if (Name.consume_front("cp.async.bulk.")) {
Intrinsic::ID ID =
StringSwitch<Intrinsic::ID>(Name)
- .Case("global.to.shared.cluster",
- Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
.Case("shared.cta.to.cluster",
Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
.Default(Intrinsic::not_intrinsic);
@@ -2076,6 +2123,26 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return true;
}
+ // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
+ // (multicast-mask overloading + trailing flag_valid_pattern).
+ SmallVector<Type *, 1> BulkG2SOvlTys;
+ IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
+ if (IID != Intrinsic::not_intrinsic) {
+ rename(F);
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
+ BulkG2SOvlTys);
+ return true;
+ }
+
+ // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
+ // (no ignore_bytes_left/right + trailing flag_valid_pattern).
+ IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
+ if (IID != Intrinsic::not_intrinsic) {
+ rename(F);
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
+ return true;
+ }
+
// Upgrade mbarrier.init intrinsics missing the layout operand.
IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
if (IID != Intrinsic::not_intrinsic) {
@@ -6046,7 +6113,42 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
CI->eraseFromParent();
return;
}
- case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
+ case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
+ SmallVector<Value *, 4> Args(CI->args());
+ unsigned AS = Args[0]->getType()->getPointerAddressSpace();
+ if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
+ Args[0] = Builder.CreateAddrSpaceCast(
+ Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
+
+ // Append the missing trailing flag_valid_pattern (0 = disabled).
+ Args.push_back(Builder.getInt32(0));
+
+ NewCall = Builder.CreateCall(NewFn, Args);
+ NewCall->takeName(CI);
+ CI->replaceAllUsesWith(NewCall);
+ CI->eraseFromParent();
+ return;
+ }
+ case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
+ // (dst, mbar, src, size, ch, flag_ch)
+ // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
+ // i32 0 /* flag_valid_pattern=disabled */)
+ SmallVector<Value *, 10> Args;
+ for (unsigned I = 0; I < 4; ++I)
+ Args.push_back(CI->getArgOperand(I));
+ Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
+ Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
+ Args.push_back(CI->getArgOperand(4)); // cache_hint
+ Args.push_back(CI->getArgOperand(5)); // flag_ch
+ Args.push_back(Builder.getInt1(false)); // flag_oob
+ Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
+
+ NewCall = Builder.CreateCall(NewFn, Args);
+ NewCall->takeName(CI);
+ CI->replaceAllUsesWith(NewCall);
+ CI->eraseFromParent();
+ return;
+ }
case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
// Create a new call with the correct address space.
SmallVector<Value *, 4> Args(CI->args());
@@ -6072,7 +6174,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
// Append the missing trailing arguments with default values (cta_group,
- // validate_pattern).
+ // flag_valid_pattern).
while (Args.size() < NewFn->getFunctionType()->getNumParams())
Args.push_back(Builder.getInt32(0));
@@ -6089,10 +6191,10 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
#undef G2S_CTA_CASE
{
SmallVector<Value *, 16> Args(CI->args());
- // Append the missing trailing validate_pattern argument with default
+ // Append the missing trailing flag_valid_pattern argument with default
// value 0.
assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
- "expected only the trailing validate_pattern to be missing");
+ "expected only the trailing flag_valid_pattern to be missing");
Args.push_back(Builder.getInt32(0));
NewCall = Builder.CreateCall(NewFn, Args);
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index d0433040fc7ab..e5a45fd5e9cd9 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -62,6 +62,16 @@ void nvvm::printMBarrierLayout(raw_ostream &OS, const Constant *ImmArgVal) {
}
}
+void nvvm::printMemScope(raw_ostream &OS, const Constant *ImmArgVal) {
+ const auto *CI = dyn_cast<ConstantInt>(ImmArgVal);
+ if (!CI || CI->getZExtValue() > static_cast<uint64_t>(MemScope::SYS)) {
+ OS << "Unknown memory scope";
+ return;
+ }
+
+ OS << getMemScopeName(static_cast<MemScope>(CI->getZExtValue()));
+}
+
void nvvm::printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal) {
if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
uint64_t Val = CI->getZExtValue();
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 4c42d512c5513..be1f92048a1ab 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7218,6 +7218,20 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"reg_count argument to nvvm.setmaxnreg must be in multiples of 8");
break;
}
+ case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta:
+ case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta_relaxed: {
+ const unsigned ArgSize = Call.arg_size();
+ const unsigned FlagValidPatternIndex = ArgSize - 1;
+ const unsigned IgnoreOOBFlagIndex = 8;
+ bool IgnoreOOB =
+ cast<ConstantInt>(Call.getArgOperand(IgnoreOOBFlagIndex))->isOne();
+ const auto *FlagValidPattern =
+ cast<ConstantInt>(Call.getArgOperand(FlagValidPatternIndex));
+ Check(!IgnoreOOB || FlagValidPattern->isZero(),
+ "flag_valid_pattern must be 0 (disabled) when ignore_oob is enabled",
+ &Call);
+ break;
+ }
case Intrinsic::experimental_convergence_entry:
case Intrinsic::experimental_convergence_anchor:
break;
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index ce6ab63eebb64..dc250c67c3bc9 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -590,6 +590,12 @@ void NVPTXInstPrinter::printTMAValidateDataFlags(const MCInst *MI, int OpNum,
<< nvvm::getTMAValidateDataPatternName(Pattern);
}
+void NVPTXInstPrinter::printMemScope(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &, raw_ostream &O) {
+ const MCOperand &MO = MI->getOperand(OpNum);
+ O << "." << nvvm::getMemScopeName(static_cast<nvvm::MemScope>(MO.getImm()));
+}
+
void NVPTXInstPrinter::printEvictPolicy(const MCInst *MI, int OpNum,
const MCSubtargetInfo &, raw_ostream &O,
StringRef Modifier) {
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index 17438cb874744..dc3e229364d0e 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -69,6 +69,8 @@ class NVPTXInstPrinter : public MCInstPrinter {
raw_ostream &O);
void printTMAValidateDataFlags(const MCInst *MI, int OpNum,
const MCSubtargetInfo &STI, raw_ostream &O);
+ void printMemScope(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
+ raw_ostream &O);
void printEvictPolicy(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
void printCallOperand(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index eb6c74df072f2..23d1ca1a6c00b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -210,6 +210,16 @@ def hasMMASparseWithMXF4NVF4Scale4xE8M0
// - tile_scatter4 mode support
def hasTMABlackwellSupport : PredOr<[SM100f, SM110f]>;
+
+def hasCpAsyncBulkClusterSupport : PredAnd<[PTX80, SM90]>;
+def hasCpAsyncBulkCTASupport : PredAnd<[PTX86, SM90]>;
+
+// Checks support for relaxed memory ordering semantics support in cp.async.bulk family.
+def hasCpAsyncBulkRelaxedSupport : PredAnd<[PTX93, PredOr<[SM90a, SM100f, SM110f]>]>;
+
+// Checks for out of bounds (OOB) support in cp.async.bulk family.
+def hasCpAsyncBulkOOBSupport : PredAnd<[PTX92, SM90]>;
+
// Checks support for conversions involving e4m3x2 and e5m2x2. These arrived
// with sm_90 in PTX 7.8, which SM90 already implies, and were extended to
// sm_89 in PTX 8.1.
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index e73d1bee2dea7..ba459b8f4695c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -524,23 +524,46 @@ let Predicates = [PTX80, SM90] in {
// TMA Async Bulk Copy Functions
//------------------------------
-class CpAsyncBulkStr<bit mc, bit ch, bit mask = 0> {
+class CpAsyncBulkStr<bits<2> mc, bit ch, bit mask = 0, bit is_relaxed = 0,
+ bit oob = 0> {
+ // Multicast kind: 0 = none, 1 = 16-bit mask, 2 = 32-bit mask.
+ string mc_str = !cond(!eq(mc, 1): ".multicast::cluster",
+ !eq(mc, 2): ".multicast::cluster::32b",
+ true: "");
+
+ defvar prefix = "cp.async.bulk";
+ defvar dir_s2g = "global.shared::cta";
+ defvar dir_g2s = "shared::cluster.global";
+ defvar dir_g2s_cta= "shared::cta.global";
+ defvar dir_c2c = "shared::cluster.shared::cta";
+ defvar completion = "mbarrier::complete_tx::bytes$flag_valid_pattern";
+
// Shared to Global memory
- string S2G = "cp.async.bulk.global.shared::cta.bulk_group"
+ string S2G = prefix # "." # dir_s2g # ".bulk_group"
# !if(ch, ".L2::cache_hint", "")
# !if(mask, ".cp_mask", "");
- // Global to Shared cluster memory
- string G2S = "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes"
- # !if(mc, ".multicast::cluster", "")
- # !if(ch, ".L2::cache_hint", "");
+ // Global to Shared cluster memory. The relaxed form carries the "${flag_scope}"
+ // qualifier and an explicit ".b128" suffix; the weak form has neither.
+ string G2S = prefix
+ # !if(is_relaxed, ".relaxed${flag_scope}", "")
+ # "." # dir_g2s
+ # "." # completion
+ # mc_str
+ # !if(ch, ".L2::cache_hint", "")
+ # !if(is_relaxed, ".b128", "");
- // Global to Shared CTA memory
- string G2S_CTA = "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes"
- # !if(ch, ".L2::cache_hint", "");
+ // Global to Shared CTA memory.
+ string G2S_CTA = prefix
+ # !if(is_relaxed, ".relaxed${flag_scope}", "")
+ # "." # dir_g2s_cta
+ # "." # completion
+ # !if(ch, ".L2::cache_hint", "")
+ # !if(oob, ".ignore_oob", "")
+ # !if(is_relaxed, ".b128", "");
// Shared CTA to Cluster memory
- string C2C = "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes";
+ string C2C = prefix # "." # dir_c2c # ".mbarrier::complete_tx::bytes";
}
multiclass CP_ASYNC_BULK_S2G_INTR<bit has_ch> {
@@ -561,44 +584,127 @@ multiclass CP_ASYNC_BULK_S2G_INTR<bit has_ch> {
defm CP_ASYNC_BULK_S2G : CP_ASYNC_BULK_S2G_INTR<has_ch = 0>;
defm CP_ASYNC_BULK_S2G_CH : CP_ASYNC_BULK_S2G_INTR<has_ch = 1>;
-multiclass CP_ASYNC_BULK_G2S_INTR<bit has_ch> {
- defvar Intr = int_nvvm_cp_async_bulk_global_to_shared_cluster;
+def TMAValidateDataFlags : Operand<i32> {
+ let PrintMethod = "printTMAValidateDataFlags";
+}
- def "" : NVPTXInst<(outs),
- (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,
- B32:$size, B16:$mask, B64:$ch),
- !if(has_ch,
- CpAsyncBulkStr<0, 1>.G2S # " [$dst], [$src], $size, [$mbar], $ch;",
- CpAsyncBulkStr<0, 0>.G2S # " [$dst], [$src], $size, [$mbar];"),
- [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i16:$mask, i64:$ch, 0, !if(has_ch, -1, 0))]>,
- Requires<[PTX80, SM90]>;
+// Matches flag_valid_pattern values 0-5. The 1-5 values are Rubin-only
+// validate patterns. 0 denotes disabled flag_valid_pattern (default).
+def tma_validate_data_imm : TImmLeaf<i32, [{
+ return Imm == 0 || (Imm >= 1 && Imm <= 5 &&
+ Subtarget->hasFeature(NVPTX::SM107f));
+}]>;
- def _MC : NVPTXInst<(outs),
- (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,
- B32:$size, B16:$mask, B64:$ch),
- !if(has_ch,
- CpAsyncBulkStr<1, 1>.G2S # " [$dst], [$src], $size, [$mbar], $mask, $ch;",
- CpAsyncBulkStr<1, 0>.G2S # " [$dst], [$src], $size, [$mbar], $mask;"),
- [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i16:$mask, i64:$ch, -1, !if(has_ch, -1, 0))]>,
- Requires<[PTX80, SM90]>;
+// Prints the flag_scope for relaxed memory ordering semantics.
+def MemScopeFlags : Operand<i32> {
+ let PrintMethod = "printMemScope";
+}
+
+// Matches flag_scope values 0-3 (cta/cluster/gpu/sys) for relaxed memory ordering semantics.
+def async_bulk_scope_imm : TImmLeaf<i32, [{ return Imm >= 0 && Imm <= 3; }]>;
+
+multiclass CP_ASYNC_BULK_G2S_INTR<bit has_ch, bit is_relaxed,
+ list<Predicate> preds = [hasCpAsyncBulkClusterSupport]> {
+ defvar Intr =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_global_to_shared_cluster" #
+ !if(is_relaxed, "_relaxed", ""));
+
+ defvar asm_nomc = CpAsyncBulkStr<0, has_ch, 0, is_relaxed>.G2S;
+ defvar asm_mc16 = CpAsyncBulkStr<1, has_ch, 0, is_relaxed>.G2S;
+ defvar asm_mc32 = CpAsyncBulkStr<2, has_ch, 0, is_relaxed>.G2S;
+
+ // Relaxed form carries the extra flag_scope operand.
+ defvar ins_tail = !if(is_relaxed,
+ (ins MemScopeFlags:$flag_scope, TMAValidateDataFlags:$flag_valid_pattern),
+ (ins TMAValidateDataFlags:$flag_valid_pattern));
+ defvar ins_i16 = !con((ins ADDR:$dst, ADDR:$mbar, ADDR:$src, B32:$size,
+ B16:$mask, B64:$ch), ins_tail);
+ defvar ins_i32 = !con((ins ADDR:$dst, ADDR:$mbar, ADDR:$src, B32:$size,
+ B32:$mask, B64:$ch), ins_tail);
+
+ // Relaxed form carries the extra flag_scope operand.
+ defvar p_tail = !if(is_relaxed,
+ (Intr async_bulk_scope_imm:$flag_scope, tma_validate_data_imm:$flag_valid_pattern),
+ (Intr tma_validate_data_imm:$flag_valid_pattern));
+ defvar p_no_mc16 = !con((Intr addr:$dst, addr:$mbar, addr:$src, i32:$size,
+ i16:$mask, i64:$ch, 0, !if(has_ch, -1, 0)), p_tail);
+ defvar p_mc16 = !con((Intr addr:$dst, addr:$mbar, addr:$src, i32:$size,
+ i16:$mask, i64:$ch, -1, !if(has_ch, -1, 0)), p_tail);
+ defvar p_no_mc32 = !con((Intr addr:$dst, addr:$mbar, addr:$src, i32:$size,
+ i32:$mask, i64:$ch, 0, !if(has_ch, -1, 0)), p_tail);
+ defvar p_mc32 = !con((Intr addr:$dst, addr:$mbar, addr:$src, i32:$size,
+ i32:$mask, i64:$ch, -1, !if(has_ch, -1, 0)), p_tail);
+
+ defvar args_ch = !if(has_ch, ", $ch", "");
+
+ // 16-bit multicast mask variants
+ def "" : NVPTXInst<(outs), ins_i16,
+ asm_nomc # " [$dst], [$src], $size, [$mbar]" # args_ch # ";",
+ [p_no_mc16]>, Requires<preds>;
+ def _MC : NVPTXInst<(outs), ins_i16,
+ asm_mc16 # " [$dst], [$src], $size, [$mbar], $mask" # args_ch # ";",
+ [p_mc16]>, Requires<preds>;
+
+ // 32-bit multicast mask variants
+ let Predicates = [hasRubinFamilySupport] in {
+ def _NO_MC32 : NVPTXInst<(outs), ins_i32,
+ asm_nomc # " [$dst], [$src], $size, [$mbar]" # args_ch # ";",
+ [p_no_mc32]>;
+ def _MC32 : NVPTXInst<(outs), ins_i32,
+ asm_mc32 # " [$dst], [$src], $size, [$mbar], $mask" # args_ch # ";",
+ [p_mc32]>;
+ }
}
-defm CP_ASYNC_BULK_G2S : CP_ASYNC_BULK_G2S_INTR<has_ch = 0>;
-defm CP_ASYNC_BULK_G2S_CH : CP_ASYNC_BULK_G2S_INTR<has_ch = 1>;
-multiclass CP_ASYNC_BULK_G2S_CTA_INTR<bit has_ch> {
- defvar Intr = int_nvvm_cp_async_bulk_global_to_shared_cta;
+defm CP_ASYNC_BULK_G2S : CP_ASYNC_BULK_G2S_INTR</*has_ch=*/0, /*is_relaxed=*/0>;
+defm CP_ASYNC_BULK_G2S_CH : CP_ASYNC_BULK_G2S_INTR</*has_ch=*/1, /*is_relaxed=*/0>;
+defm CP_ASYNC_BULK_G2S_RELAXED : CP_ASYNC_BULK_G2S_INTR</*has_ch=*/0, /*is_relaxed=*/1, [hasCpAsyncBulkRelaxedSupport]>;
+defm CP_ASYNC_BULK_G2S_RELAXED_CH : CP_ASYNC_BULK_G2S_INTR</*has_ch=*/1, /*is_relaxed=*/1, [hasCpAsyncBulkRelaxedSupport]>;
- def "" : NVPTXInst<(outs),
- (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,
- B32:$size, B64:$ch),
- !if(has_ch,
- CpAsyncBulkStr<0, 1>.G2S_CTA # " [$dst], [$src], $size, [$mbar], $ch;",
- CpAsyncBulkStr<0, 0>.G2S_CTA # " [$dst], [$src], $size, [$mbar];"),
- [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i64:$ch, !if(has_ch, -1, 0))]>,
- Requires<[PTX86, SM90]>;
+multiclass CP_ASYNC_BULK_G2S_CTA_INTR<bit has_ch, bit is_relaxed,
+ list<Predicate> preds = [hasCpAsyncBulkCTASupport]> {
+ defvar Intr =
+ !cast<Intrinsic>("int_nvvm_cp_async_bulk_global_to_shared_cta" #
+ !if(is_relaxed, "_relaxed", ""));
+
+ defvar asm = CpAsyncBulkStr<0, has_ch, 0, is_relaxed, 0>.G2S_CTA;
+ defvar asm_oob = CpAsyncBulkStr<0, has_ch, 0, is_relaxed, 1>.G2S_CTA;
+
+ // Relaxed form carries the extra flag_scope operand.
+ defvar ins_tail = !if(is_relaxed,
+ (ins MemScopeFlags:$flag_scope, TMAValidateDataFlags:$flag_valid_pattern),
+ (ins TMAValidateDataFlags:$flag_valid_pattern));
+
+ defvar ins_dag = !con((ins ADDR:$dst, ADDR:$mbar, ADDR:$src, B32:$size,
+ B64:$ch), ins_tail);
+ defvar ins_oob_dag = !con((ins ADDR:$dst, ADDR:$mbar, ADDR:$src, B32:$size,
+ B32:$ibl, B32:$ibr, B64:$ch), ins_tail);
+
+ // flag_ch/flag_oob are matched with bare constants (0 = false, -1 = true);
+ // flag_valid_pattern must be forwarded, hence the bindable TImmLeaf.
+ defvar p_tail = !if(is_relaxed,
+ (Intr async_bulk_scope_imm:$flag_scope, tma_validate_data_imm:$flag_valid_pattern),
+ (Intr tma_validate_data_imm:$flag_valid_pattern));
+ defvar p = !con((Intr addr:$dst, addr:$mbar, addr:$src, i32:$size,
+ (i32 srcvalue), (i32 srcvalue), i64:$ch, !if(has_ch, -1, 0), 0), p_tail);
+ defvar p_oob = !con((Intr addr:$dst, addr:$mbar, addr:$src, i32:$size,
+ i32:$ibl, i32:$ibr, i64:$ch, !if(has_ch, -1, 0), -1), p_tail);
+
+ defvar args_ch = !if(has_ch, ", $ch", "");
+
+ def "" : NVPTXInst<(outs), ins_dag,
+ asm # " [$dst], [$src], $size, [$mbar]" # args_ch # ";",
+ [p]>, Requires<preds>;
+
+ def _OOB : NVPTXInst<(outs), ins_oob_dag,
+ asm_oob # " [$dst], [$src], $size, $ibl, $ibr, [$mbar]" # args_ch # ";",
+ [p_oob]>, Requires<!listconcat([hasCpAsyncBulkOOBSupport], preds)>;
}
-defm CP_ASYNC_BULK_G2S_CTA : CP_ASYNC_BULK_G2S_CTA_INTR<has_ch = 0>;
-defm CP_ASYNC_BULK_G2S_CTA_CH : CP_ASYNC_BULK_G2S_CTA_INTR<has_ch = 1>;
+
+defm CP_ASYNC_BULK_G2S_CTA : CP_ASYNC_BULK_G2S_CTA_INTR</*has_ch=*/0, /*is_relaxed=*/0>;
+defm CP_ASYNC_BULK_G2S_CTA_CH : CP_ASYNC_BULK_G2S_CTA_INTR</*has_ch=*/1, /*is_relaxed=*/0>;
+defm CP_ASYNC_BULK_G2S_CTA_RELAXED : CP_ASYNC_BULK_G2S_CTA_INTR</*has_ch=*/0, /*is_relaxed=*/1, [hasCpAsyncBulkRelaxedSupport]>;
+defm CP_ASYNC_BULK_G2S_CTA_RELAXED_CH : CP_ASYNC_BULK_G2S_CTA_INTR</*has_ch=*/1, /*is_relaxed=*/1, [hasCpAsyncBulkRelaxedSupport]>;
def CP_ASYNC_BULK_CTA_TO_CLUSTER : NVPTXInst<(outs),
(ins ADDR:$dst, ADDR:$mbar, ADDR:$src, B32:$size),
@@ -677,17 +783,6 @@ def CTAGroupFlags : Operand<i32> {
def tma_cta_group_imm0 : TImmLeaf<i32, [{return Imm == 0;}]>;
def tma_cta_group_imm_any : TImmLeaf<i32, [{return Imm >= 0;}]>;
-def TMAValidateDataFlags : Operand<i32> {
- let PrintMethod = "printTMAValidateDataFlags";
-}
-
-// Matches validate pattern values 0-5. The 1-5 values are Rubin-only
-// validate patterns. 0 denotes disabled validate pattern (default).
-def tma_validate_data_imm : TImmLeaf<i32, [{
- return Imm == 0 || (Imm >= 1 && Imm <= 5 &&
- Subtarget->hasFeature(NVPTX::SM107f));
-}]>;
-
multiclass TMA_TENSOR_G2S_INTR<int dim, string mode, list<Predicate> pred,
TImmLeaf cta_group_type = tma_cta_group_imm_any> {
defvar dims_dag = TMA_DIMS_DAG_UTIL<dim>.ins_dag;
@@ -703,12 +798,12 @@ multiclass TMA_TENSOR_G2S_INTR<int dim, string mode, list<Predicate> pred,
# "." # "shared::cluster.global"
# "." # !subst("_", "::", mode)
# "." # "mbarrier::complete_tx::bytes"
- # "$validate" # "$cg";
+ # "$flag_valid_pattern" # "$cg";
defvar ins_dag_base = !con(
(ins ADDR:$dst, ADDR:$mbar, B64:$tmap),
dims_dag, im2col_dag);
- defvar ins_tail = (ins CTAGroupFlags:$cg, TMAValidateDataFlags:$validate);
+ defvar ins_tail = (ins CTAGroupFlags:$cg, TMAValidateDataFlags:$flag_valid_pattern);
defvar ins_dag = !con(ins_dag_base, ins_tail);
defvar ins_dag_ch = !con(ins_dag_base, (ins B64:$ch), ins_tail);
@@ -724,7 +819,7 @@ multiclass TMA_TENSOR_G2S_INTR<int dim, string mode, list<Predicate> pred,
(intr addr:$dst, addr:$mbar, B64:$tmap),
!setdagop(dims_dag, intr),
!setdagop(im2col_dag, intr));
- defvar intr_tail = (intr cta_group_type:$cg, tma_validate_data_imm:$validate);
+ defvar intr_tail = (intr cta_group_type:$cg, tma_validate_data_imm:$flag_valid_pattern);
defvar intr_dag_no_hints = !con(intr_dag_base, (intr (i16 srcvalue), (i64 srcvalue), 0, 0), intr_tail);
defvar intr_dag_with_mc = !con(intr_dag_base, (intr B16:$mc, (i64 srcvalue), -1, 0), intr_tail);
@@ -805,14 +900,14 @@ multiclass TMA_TENSOR_G2S_CTA_INTR<int dim, string mode, list<Predicate> pred =
# "." # "shared::cta.global"
# "." # !subst("_", "::", mode)
# "." # "mbarrier::complete_tx::bytes"
- # "$validate";
+ # "$flag_valid_pattern";
defvar ins_dag_base = !con(
(ins ADDR:$dst, ADDR:$mbar, B64:$tmap),
dims_dag, im2col_dag);
- defvar ins_dag = !con(ins_dag_base, (ins TMAValidateDataFlags:$validate));
+ defvar ins_dag = !con(ins_dag_base, (ins TMAValidateDataFlags:$flag_valid_pattern));
defvar ins_dag_ch =
- !con(ins_dag_base, (ins B64:$ch, TMAValidateDataFlags:$validate));
+ !con(ins_dag_base, (ins B64:$ch, TMAValidateDataFlags:$flag_valid_pattern));
defvar intr = !cast<Intrinsic>(
"int_nvvm_cp_async_bulk_tensor_g2s_cta_" # mode # "_" # dim_val # "d");
@@ -822,9 +917,9 @@ multiclass TMA_TENSOR_G2S_CTA_INTR<int dim, string mode, list<Predicate> pred =
!setdagop(dims_dag, intr),
!setdagop(im2col_dag, intr));
defvar intr_dag = !con(intr_dag_base,
- (intr (i64 srcvalue), 0, tma_validate_data_imm:$validate));
+ (intr (i64 srcvalue), 0, tma_validate_data_imm:$flag_valid_pattern));
defvar intr_dag_with_ch = !con(intr_dag_base,
- (intr B64:$ch, -1, tma_validate_data_imm:$validate));
+ (intr B64:$ch, -1, tma_validate_data_imm:$flag_valid_pattern));
let Predicates = pred in {
def "" : NVPTXInst<(outs), ins_dag,
@@ -1015,13 +1110,13 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_G2S_OVERRIDE_INTR<int dim, string mode, st
defvar inst_name = "cp.async.bulk.tensor." # num_dims #
"d.shared::cluster.global." # !subst("_", "::", mode) #
- ".mbarrier::complete_tx::bytes" # "$validate" # "$cg";
+ ".mbarrier::complete_tx::bytes" # "$flag_valid_pattern" # "$cg";
defvar ins_dag_base = !con(
(ins ADDR:$dst, ADDR:$mbar, B64:$tmap, B64:$override_addr),
base.tensor_size_dag, base.lower_stride_dag, base.upper_stride_dag,
base.dims_dag, base.im2col_dag);
- defvar ins_tail = (ins CTAGroupFlags:$cg, TMAValidateDataFlags:$validate);
+ defvar ins_tail = (ins CTAGroupFlags:$cg, TMAValidateDataFlags:$flag_valid_pattern);
defvar ins_dag = !con(ins_dag_base, ins_tail);
defvar ins_dag_ch = !con(ins_dag_base, (ins B64:$ch), ins_tail);
@@ -1041,7 +1136,7 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_G2S_OVERRIDE_INTR<int dim, string mode, st
!setdagop(base.upper_stride_dag, intr),
!setdagop(base.dims_dag, intr),
!setdagop(base.im2col_dag, intr));
- defvar intr_tail = (intr tma_cta_group_imm_any:$cg, tma_validate_data_imm:$validate);
+ defvar intr_tail = (intr tma_cta_group_imm_any:$cg, tma_validate_data_imm:$flag_valid_pattern);
defvar intr_dag_no_mc16_ch = !con(intr_dag_base, (intr (i16 srcvalue), (i64 srcvalue), 0, 0), intr_tail);
defvar intr_dag_with_mc16 = !con(intr_dag_base, (intr B16:$mc, (i64 srcvalue), -1, 0), intr_tail);
@@ -1093,14 +1188,14 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_G2S_CTA_OVERRIDE_INTR<int dim, string mode
defvar inst_name = "cp.async.bulk.tensor." # num_dims #
"d.shared::cta.global." # !subst("_", "::", mode) #
- ".mbarrier::complete_tx::bytes" # "$validate";
+ ".mbarrier::complete_tx::bytes" # "$flag_valid_pattern";
defvar ins_dag_base = !con(
(ins ADDR:$dst, ADDR:$mbar, B64:$tmap, B64:$override_addr),
base.tensor_size_dag, base.lower_stride_dag, base.upper_stride_dag,
base.dims_dag, base.im2col_dag);
- defvar ins_dag = !con(ins_dag_base, (ins TMAValidateDataFlags:$validate));
- defvar ins_dag_ch = !con(ins_dag_base, (ins B64:$ch, TMAValidateDataFlags:$validate));
+ defvar ins_dag = !con(ins_dag_base, (ins TMAValidateDataFlags:$flag_valid_pattern));
+ defvar ins_dag_ch = !con(ins_dag_base, (ins B64:$ch, TMAValidateDataFlags:$flag_valid_pattern));
defvar intr = !cast<Intrinsic>(
"int_nvvm_cp_async_bulk_tensor_g2s_cta_" # mode # "_" # override # "_" #
@@ -1114,8 +1209,8 @@ multiclass NVPTX_CP_ASYNC_BULK_TENSOR_G2S_CTA_OVERRIDE_INTR<int dim, string mode
!setdagop(base.dims_dag, intr),
!setdagop(base.im2col_dag, intr));
- defvar intr_dag = !con(intr_dag_base, (intr (i64 srcvalue), 0, tma_validate_data_imm:$validate));
- defvar intr_dag_with_ch = !con(intr_dag_base, (intr B64:$ch, -1, tma_validate_data_imm:$validate));
+ defvar intr_dag = !con(intr_dag_base, (intr (i64 srcvalue), 0, tma_validate_data_imm:$flag_valid_pattern));
+ defvar intr_dag_with_ch = !con(intr_dag_base, (intr B64:$ch, -1, tma_validate_data_imm:$flag_valid_pattern));
let Predicates = [hasRubinFamilySupport] in {
def "" : NVPTXInst<(outs), ins_dag,
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 390d82b4668e4..05afba0452bc8 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -113,6 +113,7 @@ declare double @llvm.nvvm.atomic.add.gen.f.sys.f64.p0(ptr, double)
declare ptr addrspace(3) @llvm.nvvm.mapa.shared.cluster(ptr addrspace(3), i32)
declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(3), ptr addrspace(3), ptr addrspace(1), i32, i16, i64, i1, i1)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3), ptr addrspace(3), ptr addrspace(1), i32, i64, i1)
declare void @llvm.nvvm.cp.async.bulk.shared.cta.to.cluster(ptr addrspace(3), ptr addrspace(3), ptr addrspace(3), i32)
declare void @llvm.nvvm.tcgen05.commit.cg1(ptr)
@@ -489,10 +490,14 @@ define void @nvvm_shared_cluster_intrinsics(ptr addrspace(3) %p0, i32 %offset) {
}
; CHECK-LABEL: @nvvm_cp_async_bulk_intrinsics
-define void @nvvm_cp_async_bulk_intrinsics(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, ptr addrspace(3) %src_shared, i32 %size) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 0, i64 0, i1 false, i1 false)
+define void @nvvm_cp_async_bulk_intrinsics(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, ptr addrspace(3) %src_shared, ptr addrspace(7) %dst_cluster, i32 %size, i16 %mc, i64 %ch) {
+; CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 0, i64 0, i1 false, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst_cluster, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 true, i1 true, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 true, i1 false, /* flag_valid_pattern=disabled */ i32 0)
; CHECK: call void @llvm.nvvm.cp.async.bulk.shared.cta.to.cluster(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr addrspace(3) %src_shared, i32 %size)
call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 0, i64 0, i1 false, i1 false)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %dst_cluster, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 true, i1 true)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i64 %ch, i1 true)
call void @llvm.nvvm.cp.async.bulk.shared.cta.to.cluster(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(3) %src_shared, i32 %size)
ret void
}
@@ -547,9 +552,9 @@ define void @nvvm_tcgen05_commit_mc_16b_intrinsics(ptr %bar, ptr addrspace(3) %b
; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_im2col
define void @nvvm_cp_async_bulk_tensor_g2s_im2col(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 0, i64 0, i1 0, i1 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 0, i64 0, i1 0, i1 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 0, i64 0, i1 0, i1 0)
@@ -558,11 +563,11 @@ define void @nvvm_cp_async_bulk_tensor_g2s_im2col(ptr addrspace(3) %d, ptr addrs
; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_tile
define void @nvvm_cp_async_bulk_tensor_g2s_tile(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %4, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %5, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %2, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %3, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %4, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %5, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 0, i1 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 0, i64 0, i1 0, i1 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 0, i64 0, i1 0, i1 0)
@@ -573,14 +578,14 @@ define void @nvvm_cp_async_bulk_tensor_g2s_tile(ptr addrspace(3) %d, ptr addrspa
; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_tile_with_cta_group
define void @nvvm_cp_async_bulk_tensor_g2s_tile_with_cta_group(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 0, i64 0, i1 0, i1 1, i32 2)
ret void
}
; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_cta_tile
define void @nvvm_cp_async_bulk_tensor_g2s_cta_tile(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i64 0, i1 true, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i64 0, i1 true, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i64 0, i1 1)
ret void
}
@@ -591,11 +596,11 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d(ptr addrspace(7
; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_im2col_with_cta_group
define void @nvvm_cp_async_bulk_tensor_g2s_im2col_with_cta_group(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i16 %mc, i64 %ch, i1 1, i1 0, i32 2)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 1, i1 1, i32 2)
ret void
}
@@ -605,9 +610,9 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspa
; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_g2s_cta
define void @nvvm_cp_async_bulk_tensor_g2s_cta(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %wHalo, i16 %wOffset, i64 %ch) {
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 true, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 true, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %wHalo, i16 %wOffset, i64 %ch, i1 1)
-; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 true, /* validate_pattern=disabled */ i32 0)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 true, /* flag_valid_pattern=disabled */ i32 0)
call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-invalid.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-invalid.ll
new file mode 100644
index 0000000000000..c17ed65fdb908
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-invalid.ll
@@ -0,0 +1,30 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) writeonly, ptr addrspace(3), ptr addrspace(1) readonly, i32, i32, i32, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 6))
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) writeonly, ptr addrspace(3), ptr addrspace(1) readonly, i32, i32, i32, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 4), i32 immarg range(i32 0, 6))
+
+define void @test_cp_async_bulk_g2s_cta(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i64 %ch) {
+ ; CHECK: immarg value 6 for arg 9 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 0, i32 6)
+
+ ; CHECK: immarg value -1 for arg 9 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 0, i32 -1)
+
+ ret void
+}
+
+define void @test_cp_async_bulk_g2s_cta_relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i64 %ch) {
+ ; CHECK: immarg value 4 for arg 9 out of range [0,4)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 0, i32 4, i32 0)
+
+ ; CHECK: immarg value -1 for arg 9 out of range [0,4)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 0, i32 -1, i32 0)
+
+ ; CHECK: immarg value 6 for arg 10 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 0, i32 0, i32 6)
+
+ ; CHECK: immarg value -1 for arg 10 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 0, i32 0, i32 -1)
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-oob.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-oob.ll
new file mode 100644
index 0000000000000..21e4a6a5208ce
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-oob.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx92 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.2 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx92 | %ptxas-verify -arch=sm_90 %}
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3), ptr addrspace(3), ptr addrspace(1), i32, i32, i32, i64, i1, i1, i32)
+
+define void @cp_async_bulk_g2s_cta_oob(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(3) %dst, i32 %size, i32 %ibl, i32 %ibr, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_cta_oob(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_cta_oob_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_cta_oob_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_cta_oob_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_cta_oob_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_g2s_cta_oob_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_g2s_cta_oob_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_cta_oob_param_6];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.ignore_oob [%rd3], [%rd1], %r1, %r2, %r3, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint.ignore_oob [%rd3], [%rd1], %r1, %r2, %r3, [%rd2], %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %ibl, i32 %ibr, i64 %ch, i1 false, i1 true, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %ibl, i32 %ibr, i64 %ch, i1 true, i1 true, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-relaxed.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-relaxed.ll
new file mode 100644
index 0000000000000..1408eb2f29e55
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-relaxed.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90a -mattr=+ptx93 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100f -mattr=+ptx93 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_110f -mattr=+ptx93 | FileCheck %s
+; RUN: %if ptxas-sm_90a && ptxas-isa-9.3 %{ llc < %s -march=nvptx64 -mcpu=sm_90a -mattr=+ptx93 | %ptxas-verify -arch=sm_90a %}
+; RUN: %if ptxas-sm_100f && ptxas-isa-9.3 %{ llc < %s -march=nvptx64 -mcpu=sm_100f -mattr=+ptx93 | %ptxas-verify -arch=sm_100f %}
+; RUN: %if ptxas-sm_110f && ptxas-isa-9.3 %{ llc < %s -march=nvptx64 -mcpu=sm_110f -mattr=+ptx93 | %ptxas-verify -arch=sm_110f %}
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3), ptr addrspace(3), ptr addrspace(1), i32, i32, i32, i64, i1, i1, i32, i32)
+
+define void @cp_async_bulk_g2s_cta_relaxed_scopes(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(3) %dst, i32 %size, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_cta_relaxed_scopes(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_cta_relaxed_scopes_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_cta_relaxed_scopes_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_cta_relaxed_scopes_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_cta_relaxed_scopes_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_cta_relaxed_scopes_param_4];
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cta.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.cluster.shared::cta.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.gpu.shared::cta.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.sys.shared::cta.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 0, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 1, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 2, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 3, i32 0)
+ ret void
+}
+
+define void @cp_async_bulk_g2s_cta_relaxed_ch(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(3) %dst, i32 %size, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_cta_relaxed_ch(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_cta_relaxed_ch_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_cta_relaxed_ch_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_cta_relaxed_ch_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_cta_relaxed_ch_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_cta_relaxed_ch_param_4];
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: cp.async.bulk.relaxed.sys.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 true, i1 false, i32 0, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 true, i1 false, i32 3, i32 0)
+ ret void
+}
+
+define void @cp_async_bulk_g2s_cta_relaxed_oob(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(3) %dst, i32 %size, i32 %ibl, i32 %ibr, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_cta_relaxed_oob(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_cta_relaxed_oob_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_cta_relaxed_oob_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_cta_relaxed_oob_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_cta_relaxed_oob_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_g2s_cta_relaxed_oob_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_g2s_cta_relaxed_oob_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_cta_relaxed_oob_param_6];
+; CHECK-NEXT: cp.async.bulk.relaxed.gpu.shared::cta.global.mbarrier::complete_tx::bytes.ignore_oob.b128 [%rd3], [%rd1], %r1, %r2, %r3, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.gpu.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint.ignore_oob.b128 [%rd3], [%rd1], %r1, %r2, %r3, [%rd2], %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %ibl, i32 %ibr, i64 %ch, i1 false, i1 true, i32 2, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %ibl, i32 %ibr, i64 %ch, i1 true, i1 true, i32 2, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-validate-data.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-validate-data.ll
new file mode 100644
index 0000000000000..6295263149e11
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-cta-validate-data.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3), ptr addrspace(3), ptr addrspace(1), i32, i32, i32, i64, i1, i1, i32)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3), ptr addrspace(3), ptr addrspace(1), i32, i32, i32, i64, i1, i1, i32, i32)
+
+define void @cp_async_bulk_g2s_cta_validate_data(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(3) %dst, i32 %size, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_cta_validate_data(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_cta_validate_data_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_cta_validate_data_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_cta_validate_data_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_cta_validate_data_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_cta_validate_data_param_4];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::80000000 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8000 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::80 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 2)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 3)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 4)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 5)
+ ret void
+}
+
+define void @cp_async_bulk_g2s_cta_validate_data_ch(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(3) %dst, i32 %size, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_cta_validate_data_ch(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_cta_validate_data_ch_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_cta_validate_data_ch_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_cta_validate_data_ch_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_cta_validate_data_ch_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_cta_validate_data_ch_param_4];
+; CHECK-NEXT: cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8000.L2::cache_hint [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8000.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.sys.shared::cta.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8000.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 true, i1 false, i32 2)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 false, i1 false, i32 0, i32 2)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 true, i1 false, i32 3, i32 2)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-invalid.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-invalid.ll
new file mode 100644
index 0000000000000..0b5082479ab5c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-invalid.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) writeonly, ptr addrspace(3), ptr addrspace(1) readonly, i32, i16, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 4), i32 immarg range(i32 0, 6))
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) writeonly, ptr addrspace(3), ptr addrspace(1) readonly, i32, i16, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 6))
+
+define void @test_cp_async_bulk_g2s_cluster(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch) {
+ ; CHECK: immarg value 4 for arg 8 out of range [0,4)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 0, i1 0, i32 4, i32 0)
+
+ ; CHECK: immarg value -1 for arg 8 out of range [0,4)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 0, i1 0, i32 -1, i32 0)
+
+ ; CHECK: immarg value 6 for arg 9 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 0, i1 0, i32 0, i32 6)
+
+ ; CHECK: immarg value -1 for arg 9 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 0, i1 0, i32 0, i32 -1)
+
+ ret void
+}
+
+define void @test_cp_async_bulk_g2s_cluster_weak_validate(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch) {
+ ; CHECK: immarg value 6 for arg 8 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 0, i1 0, i32 6)
+
+ ; CHECK: immarg value -1 for arg 8 out of range [0,6)
+ tail call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 0, i1 0, i32 -1)
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-mc32.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-mc32.ll
new file mode 100644
index 0000000000000..edd4275d2f663
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-mc32.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i32(ptr addrspace(7), ptr addrspace(3), ptr addrspace(1), i32, i32, i64, i1, i1, i32)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i32(ptr addrspace(7), ptr addrspace(3), ptr addrspace(1), i32, i32, i64, i1, i1, i32, i32)
+
+define void @cp_async_bulk_g2s_mc32(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(7) %dst, i32 %size, i32 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_mc32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_mc32_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_mc32_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_mc32_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_mc32_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_g2s_mc32_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_mc32_param_5];
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster::32b [%rd3], [%rd1], %r1, [%rd2], %r2;
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster::32b.L2::cache_hint [%rd3], [%rd1], %r1, [%rd2], %r2, %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %mc, i64 %ch, i1 false, i1 false, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %mc, i64 %ch, i1 true, i1 false, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %mc, i64 %ch, i1 true, i1 true, i32 0)
+ ret void
+}
+
+define void @cp_async_bulk_g2s_relaxed_mc32(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(7) %dst, i32 %size, i32 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_relaxed_mc32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_relaxed_mc32_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_relaxed_mc32_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_relaxed_mc32_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_relaxed_mc32_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_g2s_relaxed_mc32_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_relaxed_mc32_param_5];
+; CHECK-NEXT: cp.async.bulk.relaxed.cluster.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster::32b.b128 [%rd3], [%rd1], %r1, [%rd2], %r2;
+; CHECK-NEXT: cp.async.bulk.relaxed.cluster.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster::32b.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %r2, %rd4;
+; CHECK-NEXT: cp.async.bulk.relaxed.cluster.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8.multicast::cluster::32b.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %r2, %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %mc, i64 %ch, i1 true, i1 false, i32 1, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %mc, i64 %ch, i1 true, i1 true, i32 1, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i32(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 %mc, i64 %ch, i1 true, i1 true, i32 1, i32 4)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-relaxed.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-relaxed.ll
new file mode 100644
index 0000000000000..db904af1756c0
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-relaxed.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90a -mattr=+ptx93 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100f -mattr=+ptx93 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_110f -mattr=+ptx93 | FileCheck %s
+; RUN: %if ptxas-sm_90a && ptxas-isa-9.3 %{ llc < %s -march=nvptx64 -mcpu=sm_90a -mattr=+ptx93 | %ptxas-verify -arch=sm_90a %}
+; RUN: %if ptxas-sm_100f && ptxas-isa-9.3 %{ llc < %s -march=nvptx64 -mcpu=sm_100f -mattr=+ptx93 | %ptxas-verify -arch=sm_100f %}
+; RUN: %if ptxas-sm_110f && ptxas-isa-9.3 %{ llc < %s -march=nvptx64 -mcpu=sm_110f -mattr=+ptx93 | %ptxas-verify -arch=sm_110f %}
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7), ptr addrspace(3), ptr addrspace(1), i32, i16, i64, i1, i1, i32, i32)
+
+define void @cp_async_bulk_g2s_relaxed_scopes(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(7) %dst, i32 %size, i16 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_relaxed_scopes(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_relaxed_scopes_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_relaxed_scopes_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_relaxed_scopes_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_relaxed_scopes_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_g2s_relaxed_scopes_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_relaxed_scopes_param_5];
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cluster.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.cluster.shared::cluster.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.gpu.shared::cluster.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.sys.shared::cluster.global.mbarrier::complete_tx::bytes.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 0, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 1, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 2, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 3, i32 0)
+ ret void
+}
+
+define void @cp_async_bulk_g2s_relaxed_mc_ch(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(7) %dst, i32 %size, i16 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_relaxed_mc_ch(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_relaxed_mc_ch_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_relaxed_mc_ch_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_relaxed_mc_ch_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_relaxed_mc_ch_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_g2s_relaxed_mc_ch_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_relaxed_mc_ch_param_5];
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster.b128 [%rd3], [%rd1], %r1, [%rd2], %rs1;
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %rs1, %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 true, i1 false, i32 0, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 true, i32 0, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 true, i1 true, i32 0, i32 0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-validate-data.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-validate-data.ll
new file mode 100644
index 0000000000000..fd79a37310d9a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-g2s-validate-data.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7), ptr addrspace(3), ptr addrspace(1), i32, i16, i64, i1, i1, i32)
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7), ptr addrspace(3), ptr addrspace(1), i32, i16, i64, i1, i1, i32, i32)
+
+define void @cp_async_bulk_g2s_validate_data(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(7) %dst, i32 %size, i16 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_validate_data(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_validate_data_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_validate_data_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_validate_data_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_validate_data_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_g2s_validate_data_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_validate_data_param_5];
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::80000000 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8000.multicast::cluster.L2::cache_hint [%rd3], [%rd1], %r1, [%rd2], %rs1, %rd4;
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::80.multicast::cluster [%rd3], [%rd1], %r1, [%rd2], %rs1;
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8.L2::cache_hint [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 0)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 1)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 true, i1 true, i32 2)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 true, i1 false, i32 3)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 true, i32 4)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 5)
+ ret void
+}
+
+define void @cp_async_bulk_g2s_relaxed_validate_data(ptr addrspace(1) %src, ptr addrspace(3) %bar, ptr addrspace(7) %dst, i32 %size, i16 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_g2s_relaxed_validate_data(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_g2s_relaxed_validate_data_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_g2s_relaxed_validate_data_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_g2s_relaxed_validate_data_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_g2s_relaxed_validate_data_param_3];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_g2s_relaxed_validate_data_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_g2s_relaxed_validate_data_param_5];
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8.b128 [%rd3], [%rd1], %r1, [%rd2];
+; CHECK-NEXT: cp.async.bulk.relaxed.cta.shared::cluster.global.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8.L2::cache_hint.b128 [%rd3], [%rd1], %r1, [%rd2], %rd4;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 false, i32 0, i32 4)
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.relaxed.i16(ptr addrspace(7) %dst, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i16 %mc, i64 %ch, i1 false, i1 true, i32 0, i32 4)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-override.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-override.ll
index 7ff7bef1b25b8..7946f61fecd2c 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-override.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-override.ll
@@ -40,7 +40,7 @@ define void @cp_async_bulk_tensor_g2s_tile_override_addr(ptr addrspace(7) %dst,
tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.1d.i32(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %mc32, i64 %ch, i1 0, i1 1, i32 0, i32 0)
tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.1d.i32(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i32 %mc32, i64 %ch, i1 1, i1 1, i32 0, i32 0)
- ; Check the nonzero cta_group and a nonzero validate_pattern with its report qualifier
+ ; Check the nonzero cta_group and a nonzero flag_valid_pattern with its report qualifier
tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.override.addr.1d.i16(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, ptr addrspace(1) %override_addr, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 1, i32 2, i32 5)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-validate-data.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-validate-data.ll
index 80718135912de..ab58b9b1fa5c6 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-validate-data.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s-validate-data.ll
@@ -11,25 +11,25 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7
declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7), ptr addrspace(3), ptr, i32, i32, i32, i32, i16, i16, i16, i64, i1, i1, i32, i32)
; Check all validate_data patterns for tile mode
-define void @cp_async_bulk_tensor_g2s_validate_patterns_tile(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch) {
-; CHECK-LABEL: cp_async_bulk_tensor_g2s_validate_patterns_tile(
+define void @cp_async_bulk_tensor_g2s_flag_valid_patterns_tile(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_g2s_flag_valid_patterns_tile(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<2>;
; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_g2s_validate_patterns_tile_param_0];
-; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_g2s_validate_patterns_tile_param_1];
-; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_g2s_validate_patterns_tile_param_2];
-; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_g2s_validate_patterns_tile_param_3];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_g2s_flag_valid_patterns_tile_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_g2s_flag_valid_patterns_tile_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_g2s_flag_valid_patterns_tile_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_g2s_flag_valid_patterns_tile_param_3];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%rd1], [%rd3, {%r1}], [%rd2];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::80000000 [%rd1], [%rd3, {%r1}], [%rd2];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8000 [%rd1], [%rd3, {%r1}], [%rd2];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::80 [%rd1], [%rd3, {%r1}], [%rd2];
-; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_tensor_g2s_validate_patterns_tile_param_4];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_tensor_g2s_flag_valid_patterns_tile_param_4];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_16bytes::8 [%rd1], [%rd3, {%r1}], [%rd2];
-; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_g2s_validate_patterns_tile_param_5];
+; CHECK-NEXT: ld.param::func.b64 %rd4, [cp_async_bulk_tensor_g2s_flag_valid_patterns_tile_param_5];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd1], [%rd3, {%r1}], [%rd2];
; CHECK-NEXT: cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff.cta_group::2.multicast::cluster.L2::cache_hint [%rd1], [%rd3, {%r1}], [%rd2], %rs1, %rd4;
; CHECK-NEXT: ret;
@@ -46,27 +46,27 @@ define void @cp_async_bulk_tensor_g2s_validate_patterns_tile(ptr addrspace(7) %d
}
; Check all validate_data patterns for all load-mode layouts
-define void @cp_async_bulk_tensor_g2s_validate_patterns_modes(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %wHalo, i16 %wOffset, i64 %ch) {
-; CHECK-LABEL: cp_async_bulk_tensor_g2s_validate_patterns_modes(
+define void @cp_async_bulk_tensor_g2s_flag_valid_patterns_modes(ptr addrspace(7) %dst, ptr addrspace(3) %mbar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %wHalo, i16 %wOffset, i64 %ch) {
+; CHECK-LABEL: cp_async_bulk_tensor_g2s_flag_valid_patterns_modes(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<4>;
; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_0];
-; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_1];
-; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_2];
-; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_3];
-; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_4];
-; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_5];
-; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_6];
-; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_7];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_2];
+; CHECK-NEXT: ld.param::func.b32 %r1, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_5];
+; CHECK-NEXT: ld.param::func.b32 %r4, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r5, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_7];
; CHECK-NEXT: cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd1], [%rd3, {%r1, %r2, %r3, %r4, %r5}], [%rd2];
-; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_8];
+; CHECK-NEXT: ld.param::func.b16 %rs1, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_8];
; CHECK-NEXT: cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd1], [%rd3, {%r1, %r2, %r3}], [%rd2], {%rs1};
-; CHECK-NEXT: ld.param::func.b16 %rs2, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_9];
-; CHECK-NEXT: ld.param::func.b16 %rs3, [cp_async_bulk_tensor_g2s_validate_patterns_modes_param_10];
+; CHECK-NEXT: ld.param::func.b16 %rs2, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_9];
+; CHECK-NEXT: ld.param::func.b16 %rs3, [cp_async_bulk_tensor_g2s_flag_valid_patterns_modes_param_10];
; CHECK-NEXT: cp.async.bulk.tensor.4d.shared::cluster.global.im2col::w.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs2, %rs3};
; CHECK-NEXT: cp.async.bulk.tensor.4d.shared::cluster.global.im2col::w::128.mbarrier::complete_tx::bytes.mbarrier::report::validity::per_element::ff [%rd1], [%rd3, {%r1, %r2, %r3, %r4}], [%rd2], {%rs2, %rs3};
; CHECK-NEXT: ret;
diff --git a/llvm/test/Verifier/NVPTX/cp-async-bulk-g2s-cta.ll b/llvm/test/Verifier/NVPTX/cp-async-bulk-g2s-cta.ll
new file mode 100644
index 0000000000000..df3a01892ff43
--- /dev/null
+++ b/llvm/test/Verifier/NVPTX/cp-async-bulk-g2s-cta.ll
@@ -0,0 +1,16 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) writeonly, ptr addrspace(3), ptr addrspace(1) readonly, i32, i32, i32, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 6))
+declare void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) writeonly, ptr addrspace(3), ptr addrspace(1) readonly, i32, i32, i32, i64, i1 immarg, i1 immarg, i32 immarg range(i32 0, 4), i32 immarg range(i32 0, 6))
+
+define void @test_cp_async_bulk_g2s_cta(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i64 %ch) {
+ ; CHECK: flag_valid_pattern must be 0 (disabled) when ignore_oob is enabled
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 0, i1 1, i32 2)
+ ret void
+}
+
+define void @test_cp_async_bulk_g2s_cta_relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i64 %ch) {
+ ; CHECK: flag_valid_pattern must be 0 (disabled) when ignore_oob is enabled
+ call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta.relaxed(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr addrspace(1) %src, i32 %size, i32 0, i32 0, i64 %ch, i1 1, i1 1, i32 0, i32 4)
+ ret void
+}
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 3226063edeb88..88a6ee167bc74 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -5040,7 +5040,7 @@ def NVVM_CpAsyncBulkGlobalToSharedClusterOp :
string llvmBuilder = [{
auto [id, args] = NVVM::CpAsyncBulkGlobalToSharedClusterOp::getIntrinsicIDAndArgs(
*op, moduleTranslation, builder);
- createIntrinsicCall(builder, id, args);
+ createIntrinsicCall(builder, id, builder.getVoidTy(), args);
}];
}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 449bf7041ace9..46e9ab29bc4a0 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -4579,6 +4579,9 @@ mlir::NVVM::IDArgPair CpAsyncBulkGlobalToSharedClusterOp::getIntrinsicIDAndArgs(
llvm::Value *i16Unused = llvm::ConstantInt::get(builder.getInt16Ty(), 0);
args.push_back(hasMulticastMask ? mt.lookupValue(multicastMask)
: i16Unused);
+ } else {
+ args.push_back(builder.getInt32(0)); // ignore_bytes_left
+ args.push_back(builder.getInt32(0)); // ignore_bytes_right
}
// Cache hint, if available.
@@ -4587,10 +4590,17 @@ mlir::NVVM::IDArgPair CpAsyncBulkGlobalToSharedClusterOp::getIntrinsicIDAndArgs(
llvm::Value *i64Unused = llvm::ConstantInt::get(builder.getInt64Ty(), 0);
args.push_back(hasCacheHint ? mt.lookupValue(cacheHint) : i64Unused);
- // Flag arguments for multicast and cachehint.
- if (!isSharedCTA)
- args.push_back(builder.getInt1(hasMulticastMask));
- args.push_back(builder.getInt1(hasCacheHint));
+ // Flag arguments for multicast/ignore_oob and cachehint.
+ if (isSharedCTA) {
+ args.push_back(builder.getInt1(hasCacheHint)); // flag_ch
+ args.push_back(builder.getInt1(false)); // flag_oob
+ } else {
+ args.push_back(builder.getInt1(hasMulticastMask)); // flag_mc
+ args.push_back(builder.getInt1(hasCacheHint)); // flag_ch
+ }
+
+ // flag_valid_pattern = disabled.
+ args.push_back(builder.getInt32(0));
llvm::Intrinsic::ID id =
isSharedCTA
@@ -4679,8 +4689,8 @@ CpAsyncBulkTensorGlobalToSharedClusterOp::getIntrinsicIDAndArgs(
llvm::Value *cg =
llvm::ConstantInt::get(llvm::Type::getInt32Ty(mt.getLLVMContext()), val);
- // validate_pattern = disabled
- llvm::Value *validatePattern = builder.getInt32(0);
+ // flag_valid_pattern = disabled
+ llvm::Value *flagValidPattern = builder.getInt32(0);
if (!isCTAOnly) {
// For shared::cluster, all the arguments that we build are applicable.
@@ -4689,12 +4699,12 @@ CpAsyncBulkTensorGlobalToSharedClusterOp::getIntrinsicIDAndArgs(
args.push_back(builder.getInt1(hasMC));
args.push_back(builder.getInt1(hasCacheHint));
args.push_back(cg);
- args.push_back(validatePattern);
+ args.push_back(flagValidPattern);
} else {
// For shared::cta, only cache-hint is applicable.
args.push_back(hasCacheHint ? mt.lookupValue(cacheHint) : i64Zero);
args.push_back(builder.getInt1(hasCacheHint));
- args.push_back(validatePattern);
+ args.push_back(flagValidPattern);
}
constexpr size_t numDims = 5; // 1D to 5D
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_bulk_copy.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_bulk_copy.mlir
index 240fab5b63908..1d09dcf41bc52 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_bulk_copy.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_bulk_copy.mlir
@@ -2,10 +2,10 @@
// CHECK-LABEL: @llvm_nvvm_cp_async_bulk_global_to_shared_cluster
llvm.func @llvm_nvvm_cp_async_bulk_global_to_shared_cluster(%dst : !llvm.ptr<7>, %src : !llvm.ptr<1>, %mbar : !llvm.ptr<3>, %size : i32, %mc : i16, %ch : i64) {
- // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %[[DST:.*]], ptr addrspace(3) %[[MBAR:.*]], ptr addrspace(1) %[[SRC:.*]], i32 %[[SIZE:.*]], i16 0, i64 0, i1 false, i1 false)
- // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i16 0, i64 %[[CH:.*]], i1 false, i1 true)
- // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i16 %[[MC:.*]], i64 0, i1 true, i1 false)
- // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster(ptr addrspace(7) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i16 %[[MC]], i64 %[[CH]], i1 true, i1 true)
+ // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %[[DST:.*]], ptr addrspace(3) %[[MBAR:.*]], ptr addrspace(1) %[[SRC:.*]], i32 %[[SIZE:.*]], i16 0, i64 0, i1 false, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i16 0, i64 %[[CH:.*]], i1 false, i1 true, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i16 %[[MC:.*]], i64 0, i1 true, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cluster.i16(ptr addrspace(7) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i16 %[[MC]], i64 %[[CH]], i1 true, i1 true, /* flag_valid_pattern=disabled */ i32 0)
nvvm.cp.async.bulk.shared.cluster.global %dst, %src, %mbar, %size : !llvm.ptr<7>, !llvm.ptr<1>
nvvm.cp.async.bulk.shared.cluster.global %dst, %src, %mbar, %size l2_cache_hint = %ch : !llvm.ptr<7>, !llvm.ptr<1>
@@ -18,8 +18,8 @@ llvm.func @llvm_nvvm_cp_async_bulk_global_to_shared_cluster(%dst : !llvm.ptr<7>,
// CHECK-LABEL: @llvm_nvvm_cp_async_bulk_global_to_shared_cta
llvm.func @llvm_nvvm_cp_async_bulk_global_to_shared_cta(%dst : !llvm.ptr<3>, %src : !llvm.ptr<1>, %mbar : !llvm.ptr<3>, %size : i32, %ch : i64) {
- // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %[[DST:.*]], ptr addrspace(3) %[[MBAR:.*]], ptr addrspace(1) %[[SRC:.*]], i32 %[[SIZE:.*]], i64 0, i1 false)
- // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i64 %[[CH:.*]], i1 true)
+ // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %[[DST:.*]], ptr addrspace(3) %[[MBAR:.*]], ptr addrspace(1) %[[SRC:.*]], i32 %[[SIZE:.*]], i32 0, i32 0, i64 0, i1 false, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK: call void @llvm.nvvm.cp.async.bulk.global.to.shared.cta(ptr addrspace(3) %[[DST]], ptr addrspace(3) %[[MBAR]], ptr addrspace(1) %[[SRC]], i32 %[[SIZE]], i32 0, i32 0, i64 %[[CH:.*]], i1 true, i1 false, /* flag_valid_pattern=disabled */ i32 0)
nvvm.cp.async.bulk.shared.cluster.global %dst, %src, %mbar, %size : !llvm.ptr<3>, !llvm.ptr<1>
nvvm.cp.async.bulk.shared.cluster.global %dst, %src, %mbar, %size l2_cache_hint = %ch : !llvm.ptr<3>, !llvm.ptr<1>
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_im2col.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_im2col.mlir
index 9a5b03d1c639c..540536b06c475 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_im2col.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_im2col.mlir
@@ -2,18 +2,18 @@
llvm.func @tma_load_3d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %off0: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_im2col(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 %8, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 %8, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 %8, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 %8, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 %8, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 0, i64 %8, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] im2col[%off0] mode = im2col : !llvm.ptr<7>, !llvm.ptr
@@ -35,18 +35,18 @@ llvm.func @tma_load_3d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.
llvm.func @tma_load_4d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %off0: i16, %off1: i16, %mask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_im2col(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] im2col[%off0, %off1] multicast_mask = %mask l2_cache_hint = %cacheHint mode = im2col : !llvm.ptr<7>, !llvm.ptr
@@ -68,18 +68,18 @@ llvm.func @tma_load_4d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.
llvm.func @tma_load_5d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %off0: i16, %off1: i16, %off2: i16, %mask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_im2col(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 %12, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 %12, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 %12, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 %12, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 %12, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 %12, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 %12, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 %11, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] im2col[%off0, %off1, %off2] multicast_mask = %mask l2_cache_hint = %cacheHint mode = im2col : !llvm.ptr<7>, !llvm.ptr
@@ -101,18 +101,18 @@ llvm.func @tma_load_5d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.
llvm.func @tma_load_3d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %wHalo: i16, %wOffset: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_im2col_w(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] im2col[%wHalo, %wOffset] mode = im2col_w : !llvm.ptr<7>, !llvm.ptr
@@ -134,18 +134,18 @@ llvm.func @tma_load_3d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_4d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %wHalo: i16, %wOffset: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_im2col_w(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] im2col[%wHalo, %wOffset] mode = im2col_w : !llvm.ptr<7>, !llvm.ptr
@@ -167,18 +167,18 @@ llvm.func @tma_load_4d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_5d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %wHalo: i16, %wOffset: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_im2col_w(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] im2col[%wHalo, %wOffset] mode = im2col_w : !llvm.ptr<7>, !llvm.ptr
@@ -200,18 +200,18 @@ llvm.func @tma_load_5d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_3d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %wHalo: i16, %wOffset: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_im2col_w_128(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] im2col[%wHalo, %wOffset] mode = im2col_w_128 : !llvm.ptr<7>, !llvm.ptr
@@ -233,18 +233,18 @@ llvm.func @tma_load_3d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar:
llvm.func @tma_load_4d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %wHalo: i16, %wOffset: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_im2col_w_128(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 0, i64 %10, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i16 %9, i64 %10, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] im2col[%wHalo, %wOffset] mode = im2col_w_128 : !llvm.ptr<7>, !llvm.ptr
@@ -266,18 +266,18 @@ llvm.func @tma_load_4d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar:
llvm.func @tma_load_5d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %wHalo: i16, %wOffset: i16, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_im2col_w_128(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 0, i64 %11, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.w.128.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] im2col[%wHalo, %wOffset] mode = im2col_w_128 : !llvm.ptr<7>, !llvm.ptr
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_tile.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_tile.mlir
index f43bf109c9d4a..03e2e8ec2053d 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_tile.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_load_cluster_tile.mlir
@@ -2,18 +2,18 @@
llvm.func @tma_load_1d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_1d_all_tile(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i16 %4, i64 %5) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 %5, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 %5, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 %5, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 %5, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 %5, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 %5, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 %5, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 %5, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 %5, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 %5, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 0, i64 %5, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i16 %4, i64 %5, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0] mode = tile : !llvm.ptr<7>, !llvm.ptr
@@ -36,18 +36,18 @@ llvm.func @tma_load_1d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_2d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_2d_all_tile(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i16 %5, i64 %6) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 %6, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 %6, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 %6, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 %6, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 %6, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 %6, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 %6, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 %6, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 %6, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 %6, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 0, i64 %6, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i16 %5, i64 %6, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1] mode = tile : !llvm.ptr<7>, !llvm.ptr
@@ -70,18 +70,18 @@ llvm.func @tma_load_2d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_3d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_all_tile(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 %7, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 %7, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 %7, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 %7, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 %7, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 0, i64 %7, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] mode = tile : !llvm.ptr<7>, !llvm.ptr
@@ -104,18 +104,18 @@ llvm.func @tma_load_3d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_4d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_all_tile(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 %8, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 %8, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 %8, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 %8, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 %8, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 0, i64 %8, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i64 %8, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] mode = tile : !llvm.ptr<7>, !llvm.ptr
@@ -138,18 +138,18 @@ llvm.func @tma_load_4d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llv
llvm.func @tma_load_5d_all(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %ctamask: i16, %cache: i64) {
// CHECK-LABEL: define void @tma_load_5d_all(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] : !llvm.ptr<7>, !llvm.ptr
@@ -171,18 +171,18 @@ llvm.func @tma_load_5d_all(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr
llvm.func @tma_load_2d_tile_gather4(%tma: !llvm.ptr, %dest: !llvm.ptr<7>, %bar: !llvm.ptr<3>, %row0: i32, %col0: i32, %col1: i32, %col2: i32, %col3: i32, %ctamask: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_2d_tile_gather4(ptr %0, ptr addrspace(7) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 0, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 1, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 0, i1 false, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 0, i1 true, i1 false, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 0, i64 %9, i1 false, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.gather4.2d.i16(ptr addrspace(7) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i64 %9, i1 true, i1 true, i32 2, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%row0, %col0, %col1, %col2, %col3] mode = tile_gather4 : !llvm.ptr<7>, !llvm.ptr
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_im2col.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_im2col.mlir
index 674c4d88fd173..d02e846da1876 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_im2col.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_im2col.mlir
@@ -2,8 +2,8 @@
llvm.func @tma_load_3d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %off0: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_im2col(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i64 %7, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] im2col[%off0] mode = im2col cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -14,8 +14,8 @@ llvm.func @tma_load_3d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.
llvm.func @tma_load_4d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %off0: i16, %off1: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_im2col(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9, i1 true, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9, i1 true, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] im2col[%off0, %off1] l2_cache_hint = %cacheHint mode = im2col cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -26,8 +26,8 @@ llvm.func @tma_load_4d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.
llvm.func @tma_load_5d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %off0: i16, %off1: i16, %off2: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_im2col(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 %11, i1 true, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i16 %10, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] im2col[%off0, %off1, %off2] l2_cache_hint = %cacheHint mode = im2col cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -38,8 +38,8 @@ llvm.func @tma_load_5d_im2col(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.
llvm.func @tma_load_3d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %wHalo: i16, %wOffset: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_im2col_w(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] im2col[%wHalo, %wOffset] mode = im2col_w cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -50,8 +50,8 @@ llvm.func @tma_load_3d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_4d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %wHalo: i16, %wOffset: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_im2col_w(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] im2col[%wHalo, %wOffset] mode = im2col_w cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -62,8 +62,8 @@ llvm.func @tma_load_4d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_5d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %wHalo: i16, %wOffset: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_im2col_w(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 %10) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 %10, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 %10, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] im2col[%wHalo, %wOffset] mode = im2col_w cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -74,8 +74,8 @@ llvm.func @tma_load_5d_im2col_w(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_3d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %wHalo: i16, %wOffset: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_im2col_w_128(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %8, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] im2col[%wHalo, %wOffset] mode = im2col_w_128 cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -86,8 +86,8 @@ llvm.func @tma_load_3d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar:
llvm.func @tma_load_4d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %wHalo: i16, %wOffset: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_im2col_w_128(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i16 %7, i16 %8, i64 %9, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] im2col[%wHalo, %wOffset] mode = im2col_w_128 cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -98,8 +98,8 @@ llvm.func @tma_load_4d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar:
llvm.func @tma_load_5d_im2col_w_128(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %wHalo: i16, %wOffset: i16, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_im2col_w_128(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 %10) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 %10, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.im2col.w.128.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i16 %8, i16 %9, i64 %10, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] im2col[%wHalo, %wOffset] mode = im2col_w_128 cta_only = true : !llvm.ptr<3>, !llvm.ptr
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_tile.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_tile.mlir
index 013e4b21eea2e..53d497541d2d0 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_tile.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_load_cta_tile.mlir
@@ -2,8 +2,8 @@
llvm.func @tma_load_1d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_1d_all_tile(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i64 %4) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i64 %4, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.1d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i64 %4, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0] mode = tile cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -14,8 +14,8 @@ llvm.func @tma_load_1d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_2d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_2d_all_tile(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i64 %5) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i64 %5, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i64 %5, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1] mode = tile cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -26,8 +26,8 @@ llvm.func @tma_load_2d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_3d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_3d_all_tile(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i64 %6) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i64 %6, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.3d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i64 %6, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2] mode = tile cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -38,8 +38,8 @@ llvm.func @tma_load_3d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_4d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_4d_all_tile(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i64 %7) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i64 %7, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.4d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i64 %7, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3] mode = tile cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -50,8 +50,8 @@ llvm.func @tma_load_4d_all_tile(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llv
llvm.func @tma_load_5d_all(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_5d_all(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 %8) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 %8, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.5d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 %8, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%crd0, %crd1, %crd2, %crd3, %crd4] cta_only = true : !llvm.ptr<3>, !llvm.ptr
@@ -62,8 +62,8 @@ llvm.func @tma_load_5d_all(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr
llvm.func @tma_load_2d_tile_gather4(%tma: !llvm.ptr, %dest: !llvm.ptr<3>, %bar: !llvm.ptr<3>, %row0: i32, %col0: i32, %col1: i32, %col2: i32, %col3: i32, %cacheHint: i64) {
// CHECK-LABEL: define void @tma_load_2d_tile_gather4(ptr %0, ptr addrspace(3) %1, ptr addrspace(3) %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 %8) {
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 0, i1 false, /* validate_pattern=disabled */ i32 0)
- // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 %8, i1 true, /* validate_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 0, i1 false, /* flag_valid_pattern=disabled */ i32 0)
+ // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.g2s.cta.tile.gather4.2d(ptr addrspace(3) %1, ptr addrspace(3) %2, ptr %0, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7, i64 %8, i1 true, /* flag_valid_pattern=disabled */ i32 0)
// CHECK-NEXT: ret void
// CHECK-NEXT: }
nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tma, %bar, box[%row0, %col0, %col1, %col2, %col3] mode = tile_gather4 cta_only = true : !llvm.ptr<3>, !llvm.ptr
More information about the Mlir-commits
mailing list