[llvm] [mlir] [NVPTX] Add tcgen05.mma kind::ti16 for sm_107f + mxf4{nvf4} for sm_107a (PR #215624)
Kirill Vedernikov via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 22:20:41 PDT 2026
https://github.com/kvederni updated https://github.com/llvm/llvm-project/pull/215624
>From 0e430200f94b8d41b84f49ebbf9b263e951a211b Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Tue, 11 Aug 2026 19:29:36 +0200
Subject: [PATCH 1/7] [MLIR][NVVM][NVPTX] Add tcgen05.mma kind::ti16 for
sm_107f + mxf4{nvf4} for sm_107a
---
llvm/docs/NVPTXUsage.md | 4 +
llvm/include/llvm/IR/IntrinsicsNVVM.td | 17 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 8 +-
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 3 +
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 10 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 50 ++
.../CodeGen/NVPTX/tcgen05-mma-collector-b.ll | 109 ++++
...n05-mma-disable-output-lane-collector-b.ll | 110 ++++
.../test/CodeGen/NVPTX/tcgen05-mma-invalid.ll | 6 +-
.../NVPTX/tcgen05-mma-sp-collector-b.ll | 109 ++++
...-mma-sp-disable-output-lane-collector-b.ll | 109 ++++
.../tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll | 572 ++++++++++++++++
.../CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll | 614 ++++++++++++++++++
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 7 +-
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 33 +-
.../LLVMIR/nvvm/tcgen05-mma-invalid.mlir | 19 +-
.../LLVMIR/nvvm/tcgen05-mma-shared.mlir | 10 +
.../LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir | 10 +
.../LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir | 10 +
.../LLVMIR/nvvm/tcgen05-mma-tensor.mlir | 10 +
.../LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir | 56 +-
.../LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir | 56 +-
.../LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir | 56 +-
.../LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir | 56 +-
24 files changed, 1925 insertions(+), 119 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll
create mode 100644 llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index fef89d0a399b7..8924a44e43a8e 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -3161,6 +3161,7 @@ that are supported:
| tf32 | TF32 |
| f8f6f4 | All combinations of F8, F6, and F4 |
| i8 | Signed and Unsigned 8-bit Integers |
+| ti16 | 16-bit integer types for non-block-scale operations |
| mxf8f6f4 | MX-floating point formats |
| mxf4 | MX-floating point formats (FP4) |
| mxf4nvf4 | MXF4 + custom NVIDIA 4-bit floating point (with common scaling factor) |
@@ -3319,6 +3320,7 @@ The following tables describe the possible values of the flag arguments
| TF32 | 1 |
| F8F6F4 | 2 |
| I8 | 3 |
+| TI16 | 4 |
`%cta_group_flag` flag:
@@ -3495,6 +3497,7 @@ The following tables describes the possible values of the flag arguments
| TF32 | 1 |
| F8F6F4 | 2 |
| I8 | 3 |
+| TI16 | 4 |
The `cg1` and `cg2` suffixes select CTA group 1 and CTA group 2, respectively.
@@ -3569,6 +3572,7 @@ The following tables describes the possible values of the flag arguments
| TF32 | 1 |
| F8F6F4 | 2 |
| I8 | 3 |
+| TI16 | 4 |
`%collector_usage_b_buffer_flag`:
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 6e8a289138ed8..d99a729b72e91 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -3429,7 +3429,7 @@ foreach sparse = [0, 1] in {
defvar intrinsic_properties = !listconcat(
mma.common_intr_props,
!if(!eq(scale_d, 1), scale_d_imm_range, []),
- [Range<kind_idx, 0, !if(!eq(scale_d, 1), 2, 4)>,
+ [Range<kind_idx, 0, !if(!eq(scale_d, 1), 2, 5)>,
Range<cta_group_idx, 1, 3>,
Range<collector_a_idx, 0, !if(!eq(ashift, 1), 2, 4)>,
Range<collector_b_idx, 0, 4>,
@@ -3476,7 +3476,7 @@ foreach sparse = [0, 1] in {
defvar intrinsic_properties = !listconcat(
mma.common_intr_props,
!if(!eq(scale_d, 1), scale_d_imm_range, []),
- [Range<kind_idx, 0, !if(!eq(scale_d, 1), 2, 4)>,
+ [Range<kind_idx, 0, !if(!eq(scale_d, 1), 2, 5)>,
Range<collector_a_idx, 0, !if(!eq(ashift, 1), 2, 4)>,
Range<collector_b_idx, 0, 4>,
ArgInfo<kind_idx, [ArgName<"kind">, ImmArgPrinter<"printTcgen05MMAKind">]>,
@@ -3542,11 +3542,18 @@ foreach sparse = [0, 1] in {
llvm_i32_ty, // collector_buffer_b
llvm_i32_ty]; // collector_usage_b_op
defvar nargs = !size(args);
+ defvar kind_idx = ArgIndex<nargs>;
+ defvar collector_buffer_b_idx = ArgIndex<!add(nargs, 1)>;
+ defvar collector_usage_b_op_idx = ArgIndex<!add(nargs, 2)>;
defvar intrinsic_properties = !listconcat(
mma.common_intr_props,
- [Range<ArgIndex<nargs>, 0, 4>,
- Range<ArgIndex<!add(nargs, 1)>, 0, 4>,
- Range<ArgIndex<!add(nargs, 2)>, 0, 4>]
+ [Range<kind_idx, 0, 5>,
+ Range<collector_buffer_b_idx, 0, 4>,
+ Range<collector_usage_b_op_idx, 0, 4>,
+ ArgInfo<kind_idx, [ArgName<"kind">,
+ ImmArgPrinter<"printTcgen05MMAKind">]>,
+ ArgInfo<collector_usage_b_op_idx, [ArgName<"collector_b">,
+ ImmArgPrinter<"printTcgen05CollectorUsageOp">]>]
);
def mma.record_name:
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index f49224ac74171..e64c33db35c44 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -73,7 +73,13 @@ enum class CTAGroupKind : uint8_t {
CG_2 = 2, // cta_group::2 modifier
};
-enum class Tcgen05MMAKind : uint8_t { F16 = 0, TF32 = 1, F8F6F4 = 2, I8 = 3 };
+enum class Tcgen05MMAKind : uint8_t {
+ F16 = 0,
+ TF32 = 1,
+ F8F6F4 = 2,
+ I8 = 3,
+ TI16 = 4,
+};
enum class Tcgen05CollectorUsageOp : uint8_t {
DISCARD = 0,
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index 5c8ebc6625d77..57f00416ec030 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -42,6 +42,9 @@ void nvvm::printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal) {
case Tcgen05MMAKind::I8:
OS << "i8";
return;
+ case Tcgen05MMAKind::TI16:
+ OS << "ti16";
+ return;
}
}
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 2f837475101bf..ba6ee2c3966a6 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -6426,6 +6426,7 @@ class Tcgen05MMABase<bit IsSparse, string ASpace, string Kind, int CtaGroup,
!eq(Kind, "tf32"): 1,
!eq(Kind, "f8f6f4"): 2,
!eq(Kind, "i8"): 3,
+ !eq(Kind, "ti16"): 4,
);
bit HasCollectorBQualifier = !ne(CollectorUsageB, "discard");
@@ -6478,6 +6479,7 @@ class Tcgen05MMAInst<bit IsSparse, string ASpace, string Kind, int CtaGroup,
let Predicates = !listconcat(
!if(HasCollectorBQualifier, [hasRubinFamilySupport], []),
!cond(
+ !eq(Kind, "ti16") : [hasRubinFamilySupport],
IsScaleInputD : [hasTcgen05MMAScaleInputDImm],
!eq(Kind, "i8") : [hasTcgen05MMAI8Kind],
true : [hasTcgen05InstSupport]
@@ -6516,7 +6518,7 @@ class Tcgen05MMAInst<bit IsSparse, string ASpace, string Kind, int CtaGroup,
// tcgen05.mma
foreach sparse = [0, 1] in {
foreach space = ["tensor", "shared"] in {
- foreach kind = ["f16", "tf32", "f8f6f4", "i8"] in {
+ foreach kind = ["f16", "tf32", "f8f6f4", "i8", "ti16"] in {
foreach cta_group = [1, 2] in {
foreach collector_usage_a = ["discard", "lastuse", "fill", "use"] in {
foreach scale_input_d = !if(!or(!eq(kind, "f16"),
@@ -6584,6 +6586,7 @@ class Tcgen05MMADisableOutputLaneInst<bit IsSparse, string ASpace, string Kind,
let Predicates = !listconcat(
!if(HasCollectorBQualifier, [hasRubinFamilySupport], []),
!cond(
+ !eq(Kind, "ti16") : [hasRubinFamilySupport],
IsScaleInputD : [hasTcgen05MMAScaleInputDImm],
!eq(Kind, "i8") : [hasTcgen05MMAI8Kind],
true : [hasTcgen05InstSupport]
@@ -6648,7 +6651,7 @@ class Tcgen05MMADisableOutputLaneInst<bit IsSparse, string ASpace, string Kind,
// tcgen05.mma.disable_output_lane
foreach sparse = [0, 1] in {
foreach space = ["tensor", "shared"] in {
- foreach kind = ["f16", "tf32", "f8f6f4", "i8"] in {
+ foreach kind = ["f16", "tf32", "f8f6f4", "i8", "ti16"] in {
foreach cta_group = [1, 2] in {
foreach collector_usage_a = ["fill", "use", "lastuse", "discard"] in {
foreach scale_input_d = !if(!or(!eq(kind, "f16"),
@@ -6752,6 +6755,7 @@ class Tcgen05MMAWSInst<bit IsSparse, string ASpace, string Kind,
Tcgen05MMABase<IsSparse, ASpace, Kind, /*CtaGroup=*/ 1, CollectorUsage> {
let Predicates = !cond(
+ !eq(Kind, "ti16") : [hasRubinFamilySupport],
!eq(Kind, "i8") : [hasTcgen05MMAI8Kind],
true : [hasTcgen05InstSupport]
);
@@ -6789,7 +6793,7 @@ class Tcgen05MMAWSInst<bit IsSparse, string ASpace, string Kind,
// tcgen05.mma.ws
foreach sparse = [0, 1] in {
foreach space = ["shared", "tensor"] in {
- foreach kind = ["f16", "tf32", "f8f6f4", "i8"] in {
+ foreach kind = ["f16", "tf32", "f8f6f4", "i8", "ti16"] in {
foreach collector_buffer_b = [0, 1, 2, 3] in {
foreach collector_usage_op = ["discard", "fill", "use", "lastuse"] in {
foreach zero_col_mask = [0, 1] in {
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index b2f24c1d5d2b3..10f808a84a76a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -151,6 +151,56 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
return hasAnyFeature({NVPTX::SM100f, NVPTX::SM110f});
}
+ // Checks Rubin family extensions support.
+ // - TMA S2G im2col_w mode support
+ // - tcgen05.commit shared mem A variants.
+ bool hasRubinFamilySupport() const { return hasAnyFeature({NVPTX::SM107f}); }
+
+ // Checks tcgen05.shift instruction support.
+ bool hasTcgen05ShiftSupport() const {
+ return hasAnyFeature({NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a});
+ }
+
+ bool hasTcgen05MMAScaleInputDImm() const {
+ return hasAnyFeature({NVPTX::SM100f});
+ }
+
+ bool hasTcgen05MMAI8Kind() const {
+ return hasAnyFeature({NVPTX::SM100a, NVPTX::SM110a});
+ }
+
+ bool hasTcgen05MMASparseMxf4nvf4() const {
+ return hasFeature(NVPTX::PTX87) &&
+ hasAnyFeature({NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a, NVPTX::SM107a});
+ }
+
+ bool hasTcgen05MMASparseMxf4() const {
+ return hasAnyFeature(
+ {NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a, NVPTX::SM107a});
+ }
+
+ bool hasTcgen05LdRedSupport() const {
+ return hasFeature(NVPTX::PTX88) &&
+ hasAnyFeature({NVPTX::SM103f, NVPTX::SM110f});
+ }
+
+ bool hasReduxSyncF32() const { return hasAnyFeature({NVPTX::SM100f}); }
+
+ bool hasMMABlockScale() const { return hasAnyFeature({NVPTX::SM120f}); }
+
+ bool hasMMASparseBlockScaleF4() const {
+ return hasAnyFeature({NVPTX::SM120a, NVPTX::SM121a});
+ }
+
+ bool hasMMAWithMXF4NVF4Scale4xE8M0() const {
+ return hasFeature(NVPTX::PTX91) && hasAnyFeature({NVPTX::SM120f});
+ }
+
+ bool hasMMASparseWithMXF4NVF4Scale4xE8M0() const {
+ return hasFeature(NVPTX::PTX91) &&
+ hasAnyFeature({NVPTX::SM120a, NVPTX::SM121a});
+ }
+
// f32x2 instructions in Blackwell family
bool hasF32x2Instructions() const;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
index 0eff949aaf277..a35d074637693 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
@@ -333,6 +333,115 @@ define void @tcgen05_mma_collector_b_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr ad
ret void
}
+; ---------------------------------------------------------------------------
+; ti16, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
+; FORMAT-LABEL: define void @tcgen05_mma_collector_b_ti16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_collector_b_ti16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use.collector::b::fill [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
+; FORMAT-LABEL: define void @tcgen05_mma_collector_b_ti16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_collector_b_ti16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 0)
+ ret void
+}
+
; ---------------------------------------------------------------------------
; block_scale (section 4), mxf8f6f4, cta_group::1
; Covers: shared+tensor, representative collector_a+collector_b combos
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
index ce878e31769c3..b7d646e75d172 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
@@ -287,6 +287,116 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta2(ptr addrspa
ret void
}
+; ---------------------------------------------------------------------------
+; ti16, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_6];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ld.param.b32 %r7, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill.collector::b::use [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1), tensor
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2), tensor
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=use(3), tensor
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1), tensor.ashift
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 1, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=fill(2), collector_b=use(3), tensor
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 2, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 3, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 3, i32 0)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<12>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_6+16];
+; CHECK-NEXT: ld.param.v4.b32 {%r7, %r8, %r9, %r10}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_6];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ld.param.b32 %r11, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::use [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1), tensor
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=use(3), tensor.ashift
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 1, i32 3)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 1, i32 0)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 0)
+ ret void
+}
+
; ---------------------------------------------------------------------------
; f16 scale_d, cta_group::1
; ---------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-invalid.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-invalid.ll
index 2601701651b2d..de7b9b35fd97a 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-invalid.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-invalid.ll
@@ -1,7 +1,7 @@
; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
define void @tcgen05_mma_invalid_flag_values(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
- ; CHECK: immarg value 5 for arg 5 out of range [0,4)
+ ; CHECK: immarg value 5 for arg 5 out of range [0,5)
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 5, i32 1, i32 3)
; CHECK: immarg value 0 for arg 6 out of range [1,3)
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 3, i32 0, i32 3)
@@ -19,7 +19,7 @@ define void @tcgen05_mma_invalid_flag_values(ptr addrspace(6) %dtmem, ptr addrsp
}
define void @tcgen05_mma_disable_output_lane_invalid_flag_values(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
- ; CHECK: immarg value 5 for arg 6 out of range [0,4)
+ ; CHECK: immarg value 5 for arg 6 out of range [0,5)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 5, i32 0)
; CHECK: immarg value 5 for arg 7 out of range [0,4)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 0, i32 5)
@@ -29,7 +29,7 @@ define void @tcgen05_mma_disable_output_lane_invalid_flag_values(ptr addrspace(6
}
define void @tcgen05_mma_ws_invalid_flag_values(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
- ; CHECK: immarg value 5 for arg 5 out of range [0,4)
+ ; CHECK: immarg value 5 for arg 5 out of range [0,5)
call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 5, i32 0, i32 0)
; CHECK: immarg value 5 for arg 6 out of range [0,4)
call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 0, i32 5, i32 0)
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
index 85edfd006ab5b..3d2098113c204 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
@@ -332,6 +332,115 @@ define void @tcgen05_mma_sp_collector_b_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr
ret void
}
+; ---------------------------------------------------------------------------
+; ti16, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; CHECK-LABEL: tcgen05_mma_sp_collector_b_ti16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_collector_b_ti16_cta1_param_6];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use.collector::b::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ret;
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=fill(2), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 2, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 3, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 3, i32 0)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; CHECK-LABEL: tcgen05_mma_sp_collector_b_ti16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_collector_b_ti16_cta2_param_6];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ret;
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 2)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
+ ret void
+}
+
; ---------------------------------------------------------------------------
; scale_input_d + collector_b, f16, cta_group::1
; ---------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
index 482378e2793fe..6efe0f310ea6d 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
@@ -287,6 +287,115 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta2(ptr addr
ret void
}
+; ---------------------------------------------------------------------------
+; sp ti16, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<9>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_6];
+; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_7];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ld.param.b32 %r8, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill.collector::b::use [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1), tensor
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2), tensor
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=use(3), tensor
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1), tensor.ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 1, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=fill(2), collector_b=use(3), tensor
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 2, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 3, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 3, i32 0)
+ ; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; sp ti16, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<13>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_6];
+; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_7+16];
+; CHECK-NEXT: ld.param.v4.b32 {%r8, %r9, %r10, %r11}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_7];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ld.param.b32 %r12, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::use [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=fill(2), tensor
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=use(3), tensor.ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 1, i32 3)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 1, i32 0)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 0)
+ ret void
+}
+
; ---------------------------------------------------------------------------
; sp f16 scale_d, cta_group::1
; ---------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll
new file mode 100644
index 0000000000000..599e1a903aae9
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll
@@ -0,0 +1,572 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -o - -mcpu=sm_107a -march=nvptx64 -mattr=+ptx94 | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=FORMAT
+; RUN: %if ptxas-sm_107a && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mattr=+ptx94 -mcpu=sm_107a | %ptxas-verify -arch=sm_107a %}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_cta1_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_cta1_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_cta1_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4, .block32, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4_block32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4_block32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4_block32_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_block32_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_block32_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_block32_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_block32_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_block32_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_block32_cta1_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_block32_cta1_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_block32_cta1_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_block32_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4nvf4, .block16, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4nvf4_block16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4nvf4_block16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4nvf4_block16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4nvf4, .block32, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4nvf4_block32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4nvf4_block32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4nvf4_block32_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 1, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_cta2_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_cta2_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_cta2_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4, .block32, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4_block32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4_block32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4_block32_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_block32_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_block32_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_block32_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_block32_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_block32_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_block32_cta2_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_block32_cta2_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_block32_cta2_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_block32_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4nvf4, .block16, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4nvf4_block16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4nvf4_block16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4nvf4_block16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; block_scale (section 4), mxf4nvf4, .block32, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_mxf4nvf4_block32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_mxf4nvf4_block32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_sp_mxf4nvf4_block32_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_6];
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_7];
+; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_8];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard.collector::b::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse.collector::b::fill [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill.collector::b::use [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf4nvf4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, i32 2, i32 2, i32 3)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
new file mode 100644
index 0000000000000..648c39fede662
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
@@ -0,0 +1,614 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -o - -mcpu=sm_107f -mattr=+ptx94 -march=nvptx64 %s | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=FORMAT
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mattr=+ptx94 -mcpu=sm_107f | %ptxas-verify -arch=sm_107f %}
+
+; ---------------------------------------------------------------------------
+; ti16, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
+; FORMAT-LABEL: define void @tcgen05_mma_ti16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_ti16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_cta1_param_4];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ti16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, sparse, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_ti16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_sp_ti16_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_cta1_param_6];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_ti16_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 3, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
+; FORMAT-LABEL: define void @tcgen05_mma_ti16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_ti16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_cta2_param_4];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ti16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::fill [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::use [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 3, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, sparse, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_ti16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_sp_ti16_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_cta2_param_6];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_ti16_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 3, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, ws
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ws_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
+; FORMAT-LABEL: define void @tcgen05_mma_ws_ti16(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_ws_ti16(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_ti16_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_ti16_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_ti16_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_ti16_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_ti16_param_4];
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_ti16_param_1];
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::fill [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::use [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 0, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, disable_output_lane, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask) {
+; FORMAT-LABEL: define void @tcgen05_mma_ti16_disable_output_lane_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_ti16_disable_output_lane_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_disable_output_lane_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_disable_output_lane_cta1_param_4];
+; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_ti16_disable_output_lane_cta1_param_6];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ld.param.b32 %r7, [tcgen05_mma_ti16_disable_output_lane_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 1, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, disable_output_lane, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask) {
+; FORMAT-LABEL: define void @tcgen05_mma_ti16_disable_output_lane_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_ti16_disable_output_lane_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<12>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_disable_output_lane_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_disable_output_lane_cta2_param_4];
+; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_ti16_disable_output_lane_cta2_param_6+16];
+; CHECK-NEXT: ld.param.v4.b32 {%r7, %r8, %r9, %r10}, [tcgen05_mma_ti16_disable_output_lane_cta2_param_6];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ld.param.b32 %r11, [tcgen05_mma_ti16_disable_output_lane_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 1, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, sparse, disable_output_lane, cta_group::1
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_ti16_disable_output_lane_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_sp_ti16_disable_output_lane_cta1(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<9>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_6];
+; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_7];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ld.param.b32 %r8, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 1, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, sparse, disable_output_lane, cta_group::2
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_sp_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_ti16_disable_output_lane_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_sp_ti16_disable_output_lane_cta2(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<13>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_6];
+; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_7+16];
+; CHECK-NEXT: ld.param.v4.b32 {%r8, %r9, %r10, %r11}, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_7];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ld.param.b32 %r12, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_1];
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 1, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, ws.sp
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ws_sp_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_ws_sp_ti16(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
+; CHECK-LABEL: tcgen05_mma_ws_sp_ti16(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_sp_ti16_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_sp_ti16_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_sp_ti16_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_sp_ti16_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_sp_ti16_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_sp_ti16_param_6];
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_ws_sp_ti16_param_1];
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 0, i32 3)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, ws, collector_b::b1-b3
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ws_ti16_b1_b2_b3(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
+; FORMAT-LABEL: define void @tcgen05_mma_ws_ti16_b1_b2_b3(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 3, /* collector_b=discard */ i32 0)
+; CHECK-LABEL: tcgen05_mma_ws_ti16_b1_b2_b3(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_ti16_b1_b2_b3_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_ti16_b1_b2_b3_param_4];
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b1::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_ti16_b1_b2_b3_param_1];
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b1::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b2::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b2::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b3::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b3::discard [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 3, i32 0)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, ws, zero_col_mask
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ws_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask) {
+; FORMAT-LABEL: define void @tcgen05_mma_ws_ti16_zero_col_mask(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; CHECK-LABEL: tcgen05_mma_ws_ti16_zero_col_mask(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_ti16_zero_col_mask_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_ti16_zero_col_mask_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_ti16_zero_col_mask_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_ti16_zero_col_mask_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_ti16_zero_col_mask_param_4];
+; CHECK-NEXT: ld.param.b64 %rd3, [tcgen05_mma_ws_ti16_zero_col_mask_param_6];
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_ti16_zero_col_mask_param_1];
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r3], %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r3], %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 1)
+ ret void
+}
+
+; ---------------------------------------------------------------------------
+; ti16, ws.sp, zero_col_mask
+; ---------------------------------------------------------------------------
+
+define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask) {
+; FORMAT-LABEL: define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; CHECK-LABEL: tcgen05_mma_ws_sp_ti16_zero_col_mask(
+; CHECK: {
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_5];
+; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_2];
+; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_3];
+; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_4];
+; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_6];
+; CHECK-NEXT: ld.param.b64 %rd3, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_7];
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_1];
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: ret;
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 1)
+ ret void
+}
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 46c38bcb5475d..3ebc0c423773f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -6033,13 +6033,14 @@ def Tcgen05MMAKindI8 : I32EnumAttrCase<"I8", 3, "i8">;
def Tcgen05MMAKindMXF8F6F4 : I32EnumAttrCase<"MXF8F6F4", 4, "mxf8f6f4">;
def Tcgen05MMAKindMXF4 : I32EnumAttrCase<"MXF4", 5, "mxf4">;
def Tcgen05MMAKindMXF4NVF4 : I32EnumAttrCase<"MXF4NVF4", 6, "mxf4nvf4">;
+def Tcgen05MMAKindTI16 : I32EnumAttrCase<"TI16", 7, "ti16">;
def Tcgen05MMAKind : I32EnumAttr<
"Tcgen05MMAKind",
"tcgen05 MMA Supported Types",
[Tcgen05MMAKindF16, Tcgen05MMAKindTF32, Tcgen05MMAKindF8F6F4,
Tcgen05MMAKindI8, Tcgen05MMAKindMXF8F6F4, Tcgen05MMAKindMXF4,
- Tcgen05MMAKindMXF4NVF4]> {
+ Tcgen05MMAKindMXF4NVF4, Tcgen05MMAKindTI16]> {
let cppNamespace = "::mlir::NVVM";
let genSpecializedAttr = 0;
}
@@ -6095,8 +6096,8 @@ def Tcgen05MMACollectorOpAttr : EnumAttr<NVVM_Dialect, Tcgen05MMACollectorOp, "t
}
defvar Tcgen05MMANonBlockScaleKindList =
- [Tcgen05MMAKindF16, Tcgen05MMAKindTF32,
- Tcgen05MMAKindF8F6F4, Tcgen05MMAKindI8];
+ [Tcgen05MMAKindF16, Tcgen05MMAKindTF32, Tcgen05MMAKindF8F6F4,
+ Tcgen05MMAKindI8, Tcgen05MMAKindTI16];
defvar Tcgen05MMABlockScaleKindList =
[Tcgen05MMAKindMXF8F6F4, Tcgen05MMAKindMXF4, Tcgen05MMAKindMXF4NVF4];
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 31fed0b25990d..b07a57f08f7fd 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -5569,6 +5569,27 @@ mlir::NVVM::IDArgPair TensormapReplaceOp::getIntrinsicIDAndArgs(
// NVVM tcgen05.mma functions
//===----------------------------------------------------------------------===//
+static llvm::nvvm::Tcgen05MMAKind
+getNVVMTcgen05MMAKind(NVVM::Tcgen05MMAKind kind) {
+ switch (kind) {
+ case NVVM::Tcgen05MMAKind::F16:
+ return llvm::nvvm::Tcgen05MMAKind::F16;
+ case NVVM::Tcgen05MMAKind::TF32:
+ return llvm::nvvm::Tcgen05MMAKind::TF32;
+ case NVVM::Tcgen05MMAKind::F8F6F4:
+ return llvm::nvvm::Tcgen05MMAKind::F8F6F4;
+ case NVVM::Tcgen05MMAKind::I8:
+ return llvm::nvvm::Tcgen05MMAKind::I8;
+ case NVVM::Tcgen05MMAKind::TI16:
+ return llvm::nvvm::Tcgen05MMAKind::TI16;
+ case NVVM::Tcgen05MMAKind::MXF8F6F4:
+ case NVVM::Tcgen05MMAKind::MXF4:
+ case NVVM::Tcgen05MMAKind::MXF4NVF4:
+ llvm_unreachable("unsupported tcgen05.mma kind");
+ }
+ llvm_unreachable("unknown tcgen05.mma kind");
+}
+
mlir::NVVM::IDArgPair
Tcgen05MMAOp::getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
@@ -5700,7 +5721,8 @@ Tcgen05MMAOp::getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
if (hasDisableOutputLane)
args.push_back(DisableOutputLane);
- args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
+ args.push_back(builder.getInt32(
+ static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
if (!hasDisableOutputLane)
args.push_back(builder.getInt32(ctaGroup));
@@ -5888,7 +5910,8 @@ mlir::NVVM::IDArgPair Tcgen05MMASparseOp::getIntrinsicIDAndArgs(
if (hasDisableOutputLane)
args.push_back(DisableOutputLane);
- args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
+ args.push_back(builder.getInt32(
+ static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
if (!hasDisableOutputLane)
args.push_back(builder.getInt32(ctaGroup));
@@ -6121,7 +6144,8 @@ mlir::NVVM::IDArgPair Tcgen05MMAWsOp::getIntrinsicIDAndArgs(
ID = isATensor ? llvm::Intrinsic::nvvm_tcgen05_mma_ws_tensor
: llvm::Intrinsic::nvvm_tcgen05_mma_ws_shared;
- args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
+ args.push_back(builder.getInt32(
+ static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
args.push_back(
builder.getInt32(static_cast<unsigned>(thisOp.getCollectorBBuffer())));
args.push_back(
@@ -6162,7 +6186,8 @@ mlir::NVVM::IDArgPair Tcgen05MMAWsSparseOp::getIntrinsicIDAndArgs(
ID = isATensor ? llvm::Intrinsic::nvvm_tcgen05_mma_ws_sp_tensor
: llvm::Intrinsic::nvvm_tcgen05_mma_ws_sp_shared;
- args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
+ args.push_back(builder.getInt32(
+ static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
args.push_back(
builder.getInt32(static_cast<unsigned>(thisOp.getCollectorBBuffer())));
args.push_back(
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir
index 3dd0d7a6c9aaa..be95082009229 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir
@@ -123,7 +123,7 @@ llvm.func @nvvm_tcgen05_mma_sp_mxf4_block_scale_default(%d_tmem : !llvm.ptr<6>,
// Invalid Tcgen05MMAKind for tcgen05.mma: mxf8f6f4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_invalid_kind_mxf8f6f4
llvm.func @nvvm_tcgen05_mma_invalid_kind_mxf8f6f4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d {kind = #nvvm.tcgen05_mma_kind<mxf8f6f4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
llvm.return
}
@@ -133,7 +133,7 @@ llvm.func @nvvm_tcgen05_mma_invalid_kind_mxf8f6f4(%d_tmem : !llvm.ptr<6>, %a_des
// Invalid Tcgen05MMAKind for tcgen05.mma.sp: mxf4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_sp_invalid_kind_mxf4
llvm.func @nvvm_tcgen05_mma_sp_invalid_kind_mxf4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata {kind = #nvvm.tcgen05_mma_kind<mxf4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
llvm.return
}
@@ -143,7 +143,7 @@ llvm.func @nvvm_tcgen05_mma_sp_invalid_kind_mxf4(%d_tmem : !llvm.ptr<6>, %a_desc
// Invalid Tcgen05MMAKind for tcgen05.mma.ws: mxf4nvf4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_invalid_kind_mxf4nvf4
llvm.func @nvvm_tcgen05_mma_ws_invalid_kind_mxf4nvf4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d {kind = #nvvm.tcgen05_mma_kind<mxf4nvf4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
llvm.return
}
@@ -153,7 +153,7 @@ llvm.func @nvvm_tcgen05_mma_ws_invalid_kind_mxf4nvf4(%d_tmem : !llvm.ptr<6>, %a_
// Invalid Tcgen05MMAKind for tcgen05.mma.ws.sp: mxf8f6f4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp_invalid_kind_mxf8f6f4
llvm.func @nvvm_tcgen05_mma_ws_sp_invalid_kind_mxf8f6f4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata {kind = #nvvm.tcgen05_mma_kind<mxf8f6f4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
llvm.return
}
@@ -170,6 +170,17 @@ llvm.func @nvvm_tcgen05_mma_block_scale_invalid_kind_f16(%d_tmem : !llvm.ptr<6>,
// -----
+// Invalid Tcgen05MMAKind for tcgen05.mma.block_scale: ti16 is valid only for
+// mma/mma.sp/ws/ws.sp, not for block_scale ops.
+// CHECK-LABEL: @nvvm_tcgen05_mma_block_scale_invalid_kind_ti16
+llvm.func @nvvm_tcgen05_mma_block_scale_invalid_kind_ti16(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %scale_a: !llvm.ptr<6>, %scale_b: !llvm.ptr<6>) {
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {mxf8f6f4, mxf4, mxf4nvf4}}}
+ nvvm.tcgen05.mma.block_scale %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %scale_a, %scale_b {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, !llvm.ptr<6>)
+ llvm.return
+}
+
+// -----
+
// Invalid Tcgen05MMAKind for tcgen05.mma.sp.block_scale: tf32 is only for mma/mma.sp/ws/ws.sp.
// CHECK-LABEL: @nvvm_tcgen05_mma_sp_block_scale_invalid_kind_tf32
llvm.func @nvvm_tcgen05_mma_sp_block_scale_invalid_kind_tf32(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>, %scale_a: !llvm.ptr<6>, %scale_b: !llvm.ptr<6>) {
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir
index 51099835d73f6..c7c2a89e91c3a 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir
@@ -19,6 +19,11 @@ llvm.func @nvvm_tcgen05_mma_cta_1(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %adesc:
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
+ // ti16 enum value is 7, but maps to intrinsic kind 4.
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
+
// CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
@@ -499,6 +504,11 @@ llvm.func @nvvm_tcgen05_mma_collector_b(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1)
+ // collector_a=fill(2), collector_b=use(3), kind=ti16
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+ nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1)
+
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir
index c1907ad642bf9..a423d8ed47c02 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir
@@ -19,6 +19,11 @@ llvm.func @nvvm_tcgen05_mma_sp_cta_1(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %ades
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
+ // ti16 enum value is 7, but maps to intrinsic kind 4.
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
+
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
@@ -499,6 +504,11 @@ llvm.func @nvvm_tcgen05_mma_sp_collector_b(%d_tmem : !llvm.ptr<6>, %a_desc: i64,
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
+ // collector_a=fill(2), collector_b=use(3), kind=ti16
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+ nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
+
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir
index fba75b1164e71..aceb4d335f99e 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir
@@ -19,6 +19,11 @@ llvm.func @nvvm_tcgen05_mma_sp_cta_1(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
+ // ti16 enum value is 7, but maps to intrinsic kind 4.
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
+
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
@@ -691,6 +696,11 @@ llvm.func @nvvm_tcgen05_mma_sp_collector_b(%d_tmem : !llvm.ptr<6>, %a_tmem: !llv
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
+ // collector_a=fill(2), collector_b=use(3), kind=ti16
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+ nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
+
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir
index 8ce9065e77380..089a023b9d00f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir
@@ -19,6 +19,11 @@ llvm.func @nvvm_tcgen05_mma_cta_1(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>,
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
+ // ti16 enum value is 7, but maps to intrinsic kind 4.
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
+
// CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
@@ -690,6 +695,11 @@ llvm.func @nvvm_tcgen05_mma_collector_b(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.p
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
+ // collector_a=fill(2), collector_b=use(3), kind=ti16
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+ nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
+ {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
+
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir
index 5f1aeb05888bd..d9557f2f92d83 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir
@@ -3,61 +3,65 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws
llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -69,61 +73,65 @@ llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i6
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir
index e390e350090ad..04a19f9a58f99 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir
@@ -3,61 +3,65 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp
llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -69,61 +73,65 @@ llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc:
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_sp_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir
index f7ce5484803e9..06edb115675fe 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir
@@ -3,61 +3,65 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp
llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -69,61 +73,65 @@ llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>,
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_sp_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir
index cecbb3fbd90af..bed967280eda2 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir
@@ -3,61 +3,65 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws
llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -69,61 +73,65 @@ llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 0, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
+ nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
+ {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
+
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 0, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 1, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 2, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 3, i32 1, i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
>From f5276321261b008e744a22b4f5edf05225975cd5 Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Tue, 11 Aug 2026 23:31:32 +0200
Subject: [PATCH 2/7] [MLIR][NVVM][NVPTX] Address feedback
---
llvm/include/llvm/IR/IntrinsicsNVVM.td | 2 +
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 10 ++++
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 21 +++++++
.../CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll | 60 +++++++++----------
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 7 +--
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 33 ++--------
.../LLVMIR/nvvm/tcgen05-mma-invalid.mlir | 19 ++----
.../LLVMIR/nvvm/tcgen05-mma-shared.mlir | 10 ----
.../LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir | 10 ----
.../LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir | 10 ----
.../LLVMIR/nvvm/tcgen05-mma-tensor.mlir | 10 ----
.../LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir | 56 ++++++++---------
.../LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir | 56 ++++++++---------
.../LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir | 56 ++++++++---------
.../LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir | 56 ++++++++---------
15 files changed, 170 insertions(+), 246 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index d99a729b72e91..4357ad367d269 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -3552,6 +3552,8 @@ foreach sparse = [0, 1] in {
Range<collector_usage_b_op_idx, 0, 4>,
ArgInfo<kind_idx, [ArgName<"kind">,
ImmArgPrinter<"printTcgen05MMAKind">]>,
+ ArgInfo<collector_buffer_b_idx, [ArgName<"collector_b_buffer">,
+ ImmArgPrinter<"printTcgen05MMACollectorBBuffer">]>,
ArgInfo<collector_usage_b_op_idx, [ArgName<"collector_b">,
ImmArgPrinter<"printTcgen05CollectorUsageOp">]>]
);
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index e64c33db35c44..b38ebc3e2b309 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -88,6 +88,13 @@ enum class Tcgen05CollectorUsageOp : uint8_t {
USE = 3,
};
+enum class Tcgen05MMACollectorBBuffer : uint8_t {
+ B0 = 0,
+ B1 = 1,
+ B2 = 2,
+ B3 = 3,
+};
+
enum class TensormapElemType : uint8_t {
U8 = 0,
U16 = 1,
@@ -140,6 +147,9 @@ LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
const Constant *ImmArgVal);
+LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS,
+ const Constant *ImmArgVal);
+
LLVM_ABI void printTensormapElemType(raw_ostream &OS,
const Constant *ImmArgVal);
LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS,
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index 57f00416ec030..b5cf462dfc470 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -70,6 +70,27 @@ void nvvm::printTcgen05CollectorUsageOp(raw_ostream &OS,
}
}
+void nvvm::printTcgen05MMACollectorBBuffer(raw_ostream &OS,
+ const Constant *ImmArgVal) {
+ if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
+ uint64_t Val = CI->getZExtValue();
+ switch (static_cast<Tcgen05MMACollectorBBuffer>(Val)) {
+ case Tcgen05MMACollectorBBuffer::B0:
+ OS << "b0";
+ return;
+ case Tcgen05MMACollectorBBuffer::B1:
+ OS << "b1";
+ return;
+ case Tcgen05MMACollectorBBuffer::B2:
+ OS << "b2";
+ return;
+ case Tcgen05MMACollectorBBuffer::B3:
+ OS << "b3";
+ return;
+ }
+ }
+}
+
void nvvm::printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal) {
static constexpr StringRef TensormapElemTypes[] = {
"u8", "u16", "u32", "s32", "u64", "s64",
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
index 648c39fede662..30694fbaa0606 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
@@ -223,14 +223,14 @@ define void @tcgen05_mma_sp_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6)
define void @tcgen05_mma_ws_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
; FORMAT-LABEL: define void @tcgen05_mma_ws_ti16(
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
; CHECK-LABEL: tcgen05_mma_ws_ti16(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -445,14 +445,14 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtme
define void @tcgen05_mma_ws_sp_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
; FORMAT-LABEL: define void @tcgen05_mma_ws_sp_ti16(
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=fill */ i32 2)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
; CHECK-LABEL: tcgen05_mma_ws_sp_ti16(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -496,12 +496,12 @@ define void @tcgen05_mma_ws_sp_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %a
define void @tcgen05_mma_ws_ti16_b1_b2_b3(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d) {
; FORMAT-LABEL: define void @tcgen05_mma_ws_ti16_b1_b2_b3(
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 1, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 1, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 2, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 2, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 3, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b2 */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b2 */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b3 */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* collector_b_buffer=b3 */ i32 3, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_ws_ti16_b1_b2_b3(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -540,10 +540,10 @@ define void @tcgen05_mma_ws_ti16_b1_b2_b3(ptr addrspace(6) %dtmem, ptr addrspace
define void @tcgen05_mma_ws_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask) {
; FORMAT-LABEL: define void @tcgen05_mma_ws_ti16_zero_col_mask(
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
; CHECK-LABEL: tcgen05_mma_ws_ti16_zero_col_mask(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -579,10 +579,10 @@ define void @tcgen05_mma_ws_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addr
define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask) {
; FORMAT-LABEL: define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
-; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
; CHECK-LABEL: tcgen05_mma_ws_sp_ti16_zero_col_mask(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 3ebc0c423773f..46c38bcb5475d 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -6033,14 +6033,13 @@ def Tcgen05MMAKindI8 : I32EnumAttrCase<"I8", 3, "i8">;
def Tcgen05MMAKindMXF8F6F4 : I32EnumAttrCase<"MXF8F6F4", 4, "mxf8f6f4">;
def Tcgen05MMAKindMXF4 : I32EnumAttrCase<"MXF4", 5, "mxf4">;
def Tcgen05MMAKindMXF4NVF4 : I32EnumAttrCase<"MXF4NVF4", 6, "mxf4nvf4">;
-def Tcgen05MMAKindTI16 : I32EnumAttrCase<"TI16", 7, "ti16">;
def Tcgen05MMAKind : I32EnumAttr<
"Tcgen05MMAKind",
"tcgen05 MMA Supported Types",
[Tcgen05MMAKindF16, Tcgen05MMAKindTF32, Tcgen05MMAKindF8F6F4,
Tcgen05MMAKindI8, Tcgen05MMAKindMXF8F6F4, Tcgen05MMAKindMXF4,
- Tcgen05MMAKindMXF4NVF4, Tcgen05MMAKindTI16]> {
+ Tcgen05MMAKindMXF4NVF4]> {
let cppNamespace = "::mlir::NVVM";
let genSpecializedAttr = 0;
}
@@ -6096,8 +6095,8 @@ def Tcgen05MMACollectorOpAttr : EnumAttr<NVVM_Dialect, Tcgen05MMACollectorOp, "t
}
defvar Tcgen05MMANonBlockScaleKindList =
- [Tcgen05MMAKindF16, Tcgen05MMAKindTF32, Tcgen05MMAKindF8F6F4,
- Tcgen05MMAKindI8, Tcgen05MMAKindTI16];
+ [Tcgen05MMAKindF16, Tcgen05MMAKindTF32,
+ Tcgen05MMAKindF8F6F4, Tcgen05MMAKindI8];
defvar Tcgen05MMABlockScaleKindList =
[Tcgen05MMAKindMXF8F6F4, Tcgen05MMAKindMXF4, Tcgen05MMAKindMXF4NVF4];
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index b07a57f08f7fd..31fed0b25990d 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -5569,27 +5569,6 @@ mlir::NVVM::IDArgPair TensormapReplaceOp::getIntrinsicIDAndArgs(
// NVVM tcgen05.mma functions
//===----------------------------------------------------------------------===//
-static llvm::nvvm::Tcgen05MMAKind
-getNVVMTcgen05MMAKind(NVVM::Tcgen05MMAKind kind) {
- switch (kind) {
- case NVVM::Tcgen05MMAKind::F16:
- return llvm::nvvm::Tcgen05MMAKind::F16;
- case NVVM::Tcgen05MMAKind::TF32:
- return llvm::nvvm::Tcgen05MMAKind::TF32;
- case NVVM::Tcgen05MMAKind::F8F6F4:
- return llvm::nvvm::Tcgen05MMAKind::F8F6F4;
- case NVVM::Tcgen05MMAKind::I8:
- return llvm::nvvm::Tcgen05MMAKind::I8;
- case NVVM::Tcgen05MMAKind::TI16:
- return llvm::nvvm::Tcgen05MMAKind::TI16;
- case NVVM::Tcgen05MMAKind::MXF8F6F4:
- case NVVM::Tcgen05MMAKind::MXF4:
- case NVVM::Tcgen05MMAKind::MXF4NVF4:
- llvm_unreachable("unsupported tcgen05.mma kind");
- }
- llvm_unreachable("unknown tcgen05.mma kind");
-}
-
mlir::NVVM::IDArgPair
Tcgen05MMAOp::getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
@@ -5721,8 +5700,7 @@ Tcgen05MMAOp::getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
if (hasDisableOutputLane)
args.push_back(DisableOutputLane);
- args.push_back(builder.getInt32(
- static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
+ args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
if (!hasDisableOutputLane)
args.push_back(builder.getInt32(ctaGroup));
@@ -5910,8 +5888,7 @@ mlir::NVVM::IDArgPair Tcgen05MMASparseOp::getIntrinsicIDAndArgs(
if (hasDisableOutputLane)
args.push_back(DisableOutputLane);
- args.push_back(builder.getInt32(
- static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
+ args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
if (!hasDisableOutputLane)
args.push_back(builder.getInt32(ctaGroup));
@@ -6144,8 +6121,7 @@ mlir::NVVM::IDArgPair Tcgen05MMAWsOp::getIntrinsicIDAndArgs(
ID = isATensor ? llvm::Intrinsic::nvvm_tcgen05_mma_ws_tensor
: llvm::Intrinsic::nvvm_tcgen05_mma_ws_shared;
- args.push_back(builder.getInt32(
- static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
+ args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
args.push_back(
builder.getInt32(static_cast<unsigned>(thisOp.getCollectorBBuffer())));
args.push_back(
@@ -6186,8 +6162,7 @@ mlir::NVVM::IDArgPair Tcgen05MMAWsSparseOp::getIntrinsicIDAndArgs(
ID = isATensor ? llvm::Intrinsic::nvvm_tcgen05_mma_ws_sp_tensor
: llvm::Intrinsic::nvvm_tcgen05_mma_ws_sp_shared;
- args.push_back(builder.getInt32(
- static_cast<unsigned>(getNVVMTcgen05MMAKind(thisOp.getKind()))));
+ args.push_back(builder.getInt32(static_cast<unsigned>(thisOp.getKind())));
args.push_back(
builder.getInt32(static_cast<unsigned>(thisOp.getCollectorBBuffer())));
args.push_back(
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir
index be95082009229..3dd0d7a6c9aaa 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-invalid.mlir
@@ -123,7 +123,7 @@ llvm.func @nvvm_tcgen05_mma_sp_mxf4_block_scale_default(%d_tmem : !llvm.ptr<6>,
// Invalid Tcgen05MMAKind for tcgen05.mma: mxf8f6f4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_invalid_kind_mxf8f6f4
llvm.func @nvvm_tcgen05_mma_invalid_kind_mxf8f6f4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d {kind = #nvvm.tcgen05_mma_kind<mxf8f6f4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
llvm.return
}
@@ -133,7 +133,7 @@ llvm.func @nvvm_tcgen05_mma_invalid_kind_mxf8f6f4(%d_tmem : !llvm.ptr<6>, %a_des
// Invalid Tcgen05MMAKind for tcgen05.mma.sp: mxf4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_sp_invalid_kind_mxf4
llvm.func @nvvm_tcgen05_mma_sp_invalid_kind_mxf4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata {kind = #nvvm.tcgen05_mma_kind<mxf4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
llvm.return
}
@@ -143,7 +143,7 @@ llvm.func @nvvm_tcgen05_mma_sp_invalid_kind_mxf4(%d_tmem : !llvm.ptr<6>, %a_desc
// Invalid Tcgen05MMAKind for tcgen05.mma.ws: mxf4nvf4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_invalid_kind_mxf4nvf4
llvm.func @nvvm_tcgen05_mma_ws_invalid_kind_mxf4nvf4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d {kind = #nvvm.tcgen05_mma_kind<mxf4nvf4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
llvm.return
}
@@ -153,7 +153,7 @@ llvm.func @nvvm_tcgen05_mma_ws_invalid_kind_mxf4nvf4(%d_tmem : !llvm.ptr<6>, %a_
// Invalid Tcgen05MMAKind for tcgen05.mma.ws.sp: mxf8f6f4 is only for block_scale ops.
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp_invalid_kind_mxf8f6f4
llvm.func @nvvm_tcgen05_mma_ws_sp_invalid_kind_mxf8f6f4(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8, ti16}}}
+ // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {f16, tf32, f8f6f4, i8}}}
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata {kind = #nvvm.tcgen05_mma_kind<mxf8f6f4>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
llvm.return
}
@@ -170,17 +170,6 @@ llvm.func @nvvm_tcgen05_mma_block_scale_invalid_kind_f16(%d_tmem : !llvm.ptr<6>,
// -----
-// Invalid Tcgen05MMAKind for tcgen05.mma.block_scale: ti16 is valid only for
-// mma/mma.sp/ws/ws.sp, not for block_scale ops.
-// CHECK-LABEL: @nvvm_tcgen05_mma_block_scale_invalid_kind_ti16
-llvm.func @nvvm_tcgen05_mma_block_scale_invalid_kind_ti16(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %scale_a: !llvm.ptr<6>, %scale_b: !llvm.ptr<6>) {
- // expected-error @below {{attribute 'kind' failed to satisfy constraint: tcgen05 MMA Supported Types whose value is one of {mxf8f6f4, mxf4, mxf4nvf4}}}
- nvvm.tcgen05.mma.block_scale %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %scale_a, %scale_b {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, !llvm.ptr<6>)
- llvm.return
-}
-
-// -----
-
// Invalid Tcgen05MMAKind for tcgen05.mma.sp.block_scale: tf32 is only for mma/mma.sp/ws/ws.sp.
// CHECK-LABEL: @nvvm_tcgen05_mma_sp_block_scale_invalid_kind_tf32
llvm.func @nvvm_tcgen05_mma_sp_block_scale_invalid_kind_tf32(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>, %scale_a: !llvm.ptr<6>, %scale_b: !llvm.ptr<6>) {
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir
index c7c2a89e91c3a..51099835d73f6 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-shared.mlir
@@ -19,11 +19,6 @@ llvm.func @nvvm_tcgen05_mma_cta_1(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %adesc:
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // ti16 enum value is 7, but maps to intrinsic kind 4.
- // CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
-
// CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
@@ -504,11 +499,6 @@ llvm.func @nvvm_tcgen05_mma_collector_b(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // collector_a=fill(2), collector_b=use(3), kind=ti16
- // CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
- nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1)
-
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir
index a423d8ed47c02..c1907ad642bf9 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-shared.mlir
@@ -19,11 +19,6 @@ llvm.func @nvvm_tcgen05_mma_sp_cta_1(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %ades
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // ti16 enum value is 7, but maps to intrinsic kind 4.
- // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
-
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
@@ -504,11 +499,6 @@ llvm.func @nvvm_tcgen05_mma_sp_collector_b(%d_tmem : !llvm.ptr<6>, %a_desc: i64,
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // collector_a=fill(2), collector_b=use(3), kind=ti16
- // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
- nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
-
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir
index aceb4d335f99e..fba75b1164e71 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-sp-tensor.mlir
@@ -19,11 +19,6 @@ llvm.func @nvvm_tcgen05_mma_sp_cta_1(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // ti16 enum value is 7, but maps to intrinsic kind 4.
- // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
-
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
@@ -696,11 +691,6 @@ llvm.func @nvvm_tcgen05_mma_sp_collector_b(%d_tmem : !llvm.ptr<6>, %a_tmem: !llv
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // collector_a=fill(2), collector_b=use(3), kind=ti16
- // CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
- nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
-
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir
index 089a023b9d00f..8ce9065e77380 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-tensor.mlir
@@ -19,11 +19,6 @@ llvm.func @nvvm_tcgen05_mma_cta_1(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>,
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // ti16 enum value is 7, but maps to intrinsic kind 4.
- // CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
-
// CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
@@ -695,11 +690,6 @@ llvm.func @nvvm_tcgen05_mma_collector_b(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.p
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // collector_a=fill(2), collector_b=use(3), kind=ti16
- // CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
- nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
- {kind = #nvvm.tcgen05_mma_kind<ti16>, ctaGroup = #nvvm.cta_group<cta_1>, collectorOp = #nvvm.tcgen05_mma_collectorop<fill>, collectorOpB = #nvvm.tcgen05_mma_collectorop<use>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
-
// collector_a=use(3), collector_b=use(3)
// CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=use */ i32 3)
nvvm.tcgen05.mma %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir
index d9557f2f92d83..0cfe56066e429 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-shared.mlir
@@ -3,65 +3,61 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws
llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -73,65 +69,61 @@ llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i6
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir
index 04a19f9a58f99..e734cca4554da 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-shared.mlir
@@ -3,65 +3,61 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp
llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -73,65 +69,61 @@ llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc:
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_sp_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_desc: i64, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, i64, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_desc, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir
index 06edb115675fe..4bfd5e5fea0a9 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-sp-tensor.mlir
@@ -3,65 +3,61 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp
llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -73,65 +69,61 @@ llvm.func @nvvm_tcgen05_mma_ws_sp(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>,
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_sp_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_sp_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %spmetadata: !llvm.ptr<6>, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, !llvm.ptr<6>, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws.sp %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %spmetadata, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir
index bed967280eda2..b30a760de77ad 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tcgen05-mma-ws-tensor.mlir
@@ -3,65 +3,61 @@
// CHECK-LABEL: @nvvm_tcgen05_mma_ws
llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
@@ -73,65 +69,61 @@ llvm.func @nvvm_tcgen05_mma_ws(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b
// CHECK-LABEL: @nvvm_tcgen05_mma_ws_zero_col_mask
llvm.func @nvvm_tcgen05_mma_ws_zero_col_mask(%d_tmem : !llvm.ptr<6>, %a_tmem: !llvm.ptr<6>, %b_desc: i64, %idesc: i32, %enable_input_d: i1, %zero_col_mask: i64) {
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 0, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=ti16 */ i32 4, i32 0, /* collector_b=discard */ i32 0)
- nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
- {kind = #nvvm.tcgen05_mma_kind<ti16>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
-
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=discard */ i32 0)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=discard */ i32 0)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f16 */ i32 0, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f16>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=tf32 */ i32 1, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<tf32>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=f8f6f4 */ i32 2, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<f8f6f4>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
collectorOp = #nvvm.tcgen05_mma_collectorop<lastuse>} : (!llvm.ptr<6>, !llvm.ptr<6>, i64, i32, i1, i64)
- // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, i32 1, /* collector_b=lastuse */ i32 1)
+ // CHECK: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) {{%[0-9]+}}, ptr addrspace(6) {{%[0-9]+}}, i64 {{%[0-9]+}}, i32 {{%[0-9]+}}, i1 {{%[0-9]+}}, i64 {{%[0-9]+}}, /* kind=i8 */ i32 3, /* collector_b_buffer=b1 */ i32 1, /* collector_b=lastuse */ i32 1)
nvvm.tcgen05.mma.ws %d_tmem, %a_tmem, %b_desc, %idesc, %enable_input_d, %zero_col_mask
{kind = #nvvm.tcgen05_mma_kind<i8>,
collectorBBuffer = #nvvm.tcgen05_mma_collectorb<b1>,
>From b7a5af1a0ec7842c0976db9a17b8310115619644 Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Wed, 12 Aug 2026 13:36:01 +0200
Subject: [PATCH 3/7] [NVPTX] Update test checks after fixing merge conflicts
---
.../CodeGen/NVPTX/tcgen05-mma-collector-b.ll | 24 +--
...n05-mma-disable-output-lane-collector-b.ll | 30 +--
.../NVPTX/tcgen05-mma-sp-collector-b.ll | 28 +--
...-mma-sp-disable-output-lane-collector-b.ll | 34 ++--
.../tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll | 144 +++++++-------
.../CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll | 184 +++++++++---------
6 files changed, 222 insertions(+), 222 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
index a35d074637693..f688f0c2a9c03 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
@@ -360,15 +360,15 @@ define void @tcgen05_mma_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addr
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_collector_b_ti16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_collector_b_ti16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_collector_b_ti16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_collector_b_ti16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_collector_b_ti16_cta1_param_4];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_collector_b_ti16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
@@ -418,15 +418,15 @@ define void @tcgen05_mma_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addr
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_collector_b_ti16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_collector_b_ti16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_collector_b_ti16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_collector_b_ti16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_collector_b_ti16_cta2_param_4];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_collector_b_ti16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
index b7d646e75d172..cefa8e9bdb160 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
@@ -301,16 +301,16 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_4];
-; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_6];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
-; CHECK-NEXT: ld.param.b32 %r7, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r7, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
@@ -363,17 +363,17 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_4];
-; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_6+16];
-; CHECK-NEXT: ld.param.v4.b32 {%r7, %r8, %r9, %r10}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_6+16];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r7, %r8, %r9, %r10}, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_6];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
-; CHECK-NEXT: ld.param.b32 %r11, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r11, [tcgen05_mma_disable_output_lane_collector_b_ti16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::use [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
index 3d2098113c204..79754369c346a 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
@@ -345,16 +345,16 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_collector_b_ti16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_collector_b_ti16_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_collector_b_ti16_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_collector_b_ti16_cta1_param_6];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_collector_b_ti16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
@@ -410,16 +410,16 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_collector_b_ti16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_collector_b_ti16_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_collector_b_ti16_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_collector_b_ti16_cta2_param_6];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_collector_b_ti16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
index 6efe0f310ea6d..98ba14ddf8cfb 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
@@ -301,17 +301,17 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrsp
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_6];
-; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_6];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_7];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
-; CHECK-NEXT: ld.param.b32 %r8, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r8, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
@@ -364,18 +364,18 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrsp
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_6];
-; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_7+16];
-; CHECK-NEXT: ld.param.v4.b32 {%r8, %r9, %r10, %r11}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_6];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_7+16];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r8, %r9, %r10, %r11}, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_7];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
-; CHECK-NEXT: ld.param.b32 %r12, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r12, [tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::use [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll
index 599e1a903aae9..93ec10470dd16 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-mxf4-mxf4nvf4-kind.ll
@@ -31,18 +31,18 @@ define void @tcgen05_mma_sp_mxf4_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6)
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_cta1_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_cta1_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_cta1_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4_cta1_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4_cta1_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -102,18 +102,18 @@ define void @tcgen05_mma_sp_mxf4_block32_cta1(ptr addrspace(6) %dtmem, ptr addrs
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_block32_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4_block32_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_block32_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_block32_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_block32_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_block32_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_block32_cta1_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_block32_cta1_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_block32_cta1_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4_block32_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4_block32_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4_block32_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4_block32_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4_block32_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4_block32_cta1_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4_block32_cta1_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_block32_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4_block32_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -173,18 +173,18 @@ define void @tcgen05_mma_sp_mxf4nvf4_block16_cta1(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -244,18 +244,18 @@ define void @tcgen05_mma_sp_mxf4nvf4_block32_cta1(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block32_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -315,18 +315,18 @@ define void @tcgen05_mma_sp_mxf4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6)
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_cta2_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_cta2_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_cta2_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4_cta2_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4_cta2_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -386,18 +386,18 @@ define void @tcgen05_mma_sp_mxf4_block32_cta2(ptr addrspace(6) %dtmem, ptr addrs
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4_block32_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4_block32_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4_block32_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4_block32_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4_block32_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4_block32_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4_block32_cta2_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4_block32_cta2_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4_block32_cta2_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4_block32_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4_block32_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4_block32_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4_block32_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4_block32_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4_block32_cta2_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4_block32_cta2_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4_block32_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4_block32_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -457,18 +457,18 @@ define void @tcgen05_mma_sp_mxf4nvf4_block16_cta2(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
@@ -528,18 +528,18 @@ define void @tcgen05_mma_sp_mxf4nvf4_block32_cta2(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_6];
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_7];
-; CHECK-NEXT: ld.param.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_8];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r5, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_8];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
-; CHECK-NEXT: ld.param.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r6, [tcgen05_mma_sp_mxf4nvf4_block32_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, [%r4], [%r5], %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%r1], [%r6], %rd2, [%r3], %r2, [%r4], [%r5], %p1;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
index 30694fbaa0606..07d32eac8f0a4 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
@@ -26,15 +26,15 @@ define void @tcgen05_mma_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %at
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ti16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ti16_cta1_param_4];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ti16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ti16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, %p1;
@@ -79,16 +79,16 @@ define void @tcgen05_mma_sp_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6)
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_ti16_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_ti16_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_ti16_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_ti16_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_ti16_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_ti16_cta1_param_6];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_ti16_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_ti16_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
@@ -133,15 +133,15 @@ define void @tcgen05_mma_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %at
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ti16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ti16_cta2_param_4];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ti16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ti16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, %p1;
@@ -186,16 +186,16 @@ define void @tcgen05_mma_sp_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6)
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_ti16_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_ti16_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_ti16_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_ti16_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_ti16_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_ti16_cta2_param_6];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_sp_ti16_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_sp_ti16_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
@@ -239,15 +239,15 @@ define void @tcgen05_mma_ws_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %aten
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_ti16_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ws_ti16_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_ti16_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_ti16_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_ti16_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_ti16_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ws_ti16_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ws_ti16_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ws_ti16_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ws_ti16_param_4];
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_ti16_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ws_ti16_param_1];
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r3], %rd2, %r2, %p1;
@@ -286,16 +286,16 @@ define void @tcgen05_mma_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtmem,
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_disable_output_lane_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_disable_output_lane_cta1_param_4];
-; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_ti16_disable_output_lane_cta1_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ti16_disable_output_lane_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ti16_disable_output_lane_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ti16_disable_output_lane_cta1_param_4];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_ti16_disable_output_lane_cta1_param_6];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
-; CHECK-NEXT: ld.param.b32 %r7, [tcgen05_mma_ti16_disable_output_lane_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r7, [tcgen05_mma_ti16_disable_output_lane_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
@@ -328,17 +328,17 @@ define void @tcgen05_mma_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtmem,
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ti16_disable_output_lane_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ti16_disable_output_lane_cta2_param_4];
-; CHECK-NEXT: ld.param.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_ti16_disable_output_lane_cta2_param_6+16];
-; CHECK-NEXT: ld.param.v4.b32 {%r7, %r8, %r9, %r10}, [tcgen05_mma_ti16_disable_output_lane_cta2_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ti16_disable_output_lane_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ti16_disable_output_lane_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ti16_disable_output_lane_cta2_param_4];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r3, %r4, %r5, %r6}, [tcgen05_mma_ti16_disable_output_lane_cta2_param_6+16];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r7, %r8, %r9, %r10}, [tcgen05_mma_ti16_disable_output_lane_cta2_param_6];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
-; CHECK-NEXT: ld.param.b32 %r11, [tcgen05_mma_ti16_disable_output_lane_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r11, [tcgen05_mma_ti16_disable_output_lane_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
@@ -371,17 +371,17 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtme
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_6];
-; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_6];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_7];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
-; CHECK-NEXT: ld.param.b32 %r8, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r8, [tcgen05_mma_sp_ti16_disable_output_lane_cta1_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
@@ -414,18 +414,18 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtme
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_6];
-; CHECK-NEXT: ld.param.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_7+16];
-; CHECK-NEXT: ld.param.v4.b32 {%r8, %r9, %r10, %r11}, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_6];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r4, %r5, %r6, %r7}, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_7+16];
+; CHECK-NEXT: ld.param::func.v4.b32 {%r8, %r9, %r10, %r11}, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_7];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
-; CHECK-NEXT: ld.param.b32 %r12, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r12, [tcgen05_mma_sp_ti16_disable_output_lane_cta2_param_1];
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
@@ -461,16 +461,16 @@ define void @tcgen05_mma_ws_sp_ti16(ptr addrspace(6) %dtmem, ptr addrspace(6) %a
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_sp_ti16_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ws_sp_ti16_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_sp_ti16_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_sp_ti16_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_sp_ti16_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_sp_ti16_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_sp_ti16_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ws_sp_ti16_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ws_sp_ti16_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ws_sp_ti16_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ws_sp_ti16_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ws_sp_ti16_param_6];
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_ws_sp_ti16_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_ws_sp_ti16_param_1];
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
@@ -510,15 +510,15 @@ define void @tcgen05_mma_ws_ti16_b1_b2_b3(ptr addrspace(6) %dtmem, ptr addrspace
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_ti16_b1_b2_b3_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_ti16_b1_b2_b3_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ws_ti16_b1_b2_b3_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ws_ti16_b1_b2_b3_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ws_ti16_b1_b2_b3_param_4];
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b1::discard [%r1], %rd1, %rd2, %r2, %p1;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_ti16_b1_b2_b3_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ws_ti16_b1_b2_b3_param_1];
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b1::discard [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b2::discard [%r1], %rd1, %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b2::discard [%r1], [%r3], %rd2, %r2, %p1;
@@ -552,16 +552,16 @@ define void @tcgen05_mma_ws_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addr
; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_ti16_zero_col_mask_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ws_ti16_zero_col_mask_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_ti16_zero_col_mask_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_ti16_zero_col_mask_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_ti16_zero_col_mask_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_ti16_zero_col_mask_param_4];
-; CHECK-NEXT: ld.param.b64 %rd3, [tcgen05_mma_ws_ti16_zero_col_mask_param_6];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ws_ti16_zero_col_mask_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ws_ti16_zero_col_mask_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ws_ti16_zero_col_mask_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ws_ti16_zero_col_mask_param_4];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [tcgen05_mma_ws_ti16_zero_col_mask_param_6];
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, %r2, %p1, %rd3;
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_ti16_zero_col_mask_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ws_ti16_zero_col_mask_param_1];
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r3], %rd2, %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r3], %rd2, %r2, %p1, %rd3;
@@ -591,17 +591,17 @@ define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b8 %rs1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_5];
+; CHECK-NEXT: ld.param::func.b8 %rs1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_5];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_0];
-; CHECK-NEXT: ld.param.b64 %rd1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_2];
-; CHECK-NEXT: ld.param.b64 %rd2, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_3];
-; CHECK-NEXT: ld.param.b32 %r2, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_4];
-; CHECK-NEXT: ld.param.b32 %r3, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_6];
-; CHECK-NEXT: ld.param.b64 %rd3, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_7];
+; CHECK-NEXT: ld.param::func.b32 %r1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_2];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_3];
+; CHECK-NEXT: ld.param::func.b32 %r2, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_4];
+; CHECK-NEXT: ld.param::func.b32 %r3, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_6];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_7];
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
-; CHECK-NEXT: ld.param.b32 %r4, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_1];
+; CHECK-NEXT: ld.param::func.b32 %r4, [tcgen05_mma_ws_sp_ti16_zero_col_mask_param_1];
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
>From c635436efea6cc54cab74e6925ce9936d991d5b0 Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Wed, 12 Aug 2026 13:59:17 +0200
Subject: [PATCH 4/7] [NVPTX] clang format
---
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 10f808a84a76a..e1576e04f372f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -171,7 +171,8 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
bool hasTcgen05MMASparseMxf4nvf4() const {
return hasFeature(NVPTX::PTX87) &&
- hasAnyFeature({NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a, NVPTX::SM107a});
+ hasAnyFeature(
+ {NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a, NVPTX::SM107a});
}
bool hasTcgen05MMASparseMxf4() const {
>From a9817129eed657cfd0f1be916a009b73c9a39004 Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Wed, 12 Aug 2026 16:56:05 +0200
Subject: [PATCH 5/7] [NVPTX] Add FORMAT checks
---
...n05-mma-disable-output-lane-collector-b.ll | 95 ++++++++++++
.../NVPTX/tcgen05-mma-sp-collector-b.ll | 144 ++++++++++++++++++
...-mma-sp-disable-output-lane-collector-b.ll | 91 +++++++++++
3 files changed, 330 insertions(+)
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
index cefa8e9bdb160..3fe4f0b904682 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -o - -mcpu=sm_107f -mattr=+ptx94 -march=nvptx64 %s | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=FORMAT
; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mattr=+ptx94 -mcpu=sm_107f | %ptxas-verify -arch=sm_107f %}
; ---------------------------------------------------------------------------
@@ -8,6 +9,19 @@
define void @tcgen05_mma_disable_output_lane_collector_b_f16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
; kind=f16(0), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_f16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_f16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -70,6 +84,14 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f16_cta1(ptr addrspace(
define void @tcgen05_mma_disable_output_lane_collector_b_f16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
; kind=f16(0), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_f16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_f16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -118,6 +140,14 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f16_cta2(ptr addrspace(
define void @tcgen05_mma_disable_output_lane_collector_b_tf32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
; kind=tf32(1), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_tf32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_tf32_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -165,6 +195,12 @@ define void @tcgen05_mma_disable_output_lane_collector_b_tf32_cta1(ptr addrspace
define void @tcgen05_mma_disable_output_lane_collector_b_tf32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
; kind=tf32(1), cta_group=2, collector_a=discard(0), collector_b=fill(2)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_tf32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_tf32_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -207,6 +243,13 @@ define void @tcgen05_mma_disable_output_lane_collector_b_tf32_cta2(ptr addrspace
define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
; kind=f8f6f4(2), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -251,6 +294,12 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta1(ptr addrspa
define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
; kind=f8f6f4(2), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -293,6 +342,19 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f8f6f4_cta2(ptr addrspa
define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -355,6 +417,14 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace
define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -403,6 +473,13 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace
define void @tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
; kind=f16(0), scale_d, cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 5, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 3, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 7, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 2, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 2, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 5, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -447,6 +524,12 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta1(ptr ad
define void @tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4) {
; kind=tf32(1), scale_d, cta_group=1, collector_a=discard(0), collector_b=fill(2)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 4, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 6, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 1, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 1, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 4, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -488,6 +571,12 @@ define void @tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta1(ptr a
define void @tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
; kind=f16(0), scale_d, cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 5, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 3, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 7, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 7, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 5, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -530,6 +619,12 @@ define void @tcgen05_mma_disable_output_lane_collector_b_f16_scale_d_cta2(ptr ad
define void @tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8) {
; kind=tf32(1), scale_d, cta_group=2, collector_a=discard(0), collector_b=fill(2)
+; FORMAT-LABEL: define void @tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 4, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 6, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 1, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 1, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 4, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_tf32_scale_d_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
index 79754369c346a..c4e9a738ff9d9 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -o - -mcpu=sm_107f -mattr=+ptx94 -march=nvptx64 %s | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=FORMAT
; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mattr=+ptx94 -mcpu=sm_107f | %ptxas-verify -arch=sm_107f %}
; ---------------------------------------------------------------------------
@@ -7,6 +8,20 @@
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_f16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_f16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_f16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -72,6 +87,13 @@ define void @tcgen05_mma_sp_collector_b_f16_cta1(ptr addrspace(6) %dtmem, ptr ad
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_f16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_f16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_f16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -116,6 +138,18 @@ define void @tcgen05_mma_sp_collector_b_f16_cta2(ptr addrspace(6) %dtmem, ptr ad
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_tf32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_tf32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_tf32_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -175,6 +209,18 @@ define void @tcgen05_mma_sp_collector_b_tf32_cta1(ptr addrspace(6) %dtmem, ptr a
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_f8f6f4_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_f8f6f4_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_f8f6f4_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -234,6 +280,13 @@ define void @tcgen05_mma_sp_collector_b_f8f6f4_cta1(ptr addrspace(6) %dtmem, ptr
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_tf32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_tf32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_tf32_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -278,6 +331,18 @@ define void @tcgen05_mma_sp_collector_b_tf32_cta2(ptr addrspace(6) %dtmem, ptr a
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_f8f6f4_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_f8f6f4_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -337,6 +402,20 @@ define void @tcgen05_mma_sp_collector_b_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_ti16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -402,6 +481,13 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr a
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_ti16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -446,6 +532,15 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr a
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_scale_d_f16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_scale_d_f16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_scale_d_f16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -496,6 +591,12 @@ define void @tcgen05_mma_sp_collector_b_scale_d_f16_cta1(ptr addrspace(6) %dtmem
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_scale_d_f16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_scale_d_f16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=f16 */ i32 0, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_scale_d_f16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -537,6 +638,13 @@ define void @tcgen05_mma_sp_collector_b_scale_d_f16_cta2(ptr addrspace(6) %dtmem
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_scale_d_tf32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_scale_d_tf32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_scale_d_tf32_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -581,6 +689,12 @@ define void @tcgen05_mma_sp_collector_b_scale_d_tf32_cta1(ptr addrspace(6) %dtme
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_scale_d_tf32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_scale_d_tf32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 0, /* kind=tf32 */ i32 1, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_scale_d_tf32_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -622,6 +736,17 @@ define void @tcgen05_mma_sp_collector_b_scale_d_tf32_cta2(ptr addrspace(6) %dtme
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -680,6 +805,13 @@ define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta1(ptr addrspace(
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -726,6 +858,12 @@ define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_cta2(ptr addrspace(
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -769,6 +907,12 @@ define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta1(ptr ad
; ---------------------------------------------------------------------------
define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b) {
+; FORMAT-LABEL: define void @tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.mxf8f6f4.block_scale.block32(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, ptr addrspace(6) %scale_a, ptr addrspace(6) %scale_b, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_block_scale_mxf8f6f4_block32_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
index 98ba14ddf8cfb..476e5761d24b4 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -o - -mcpu=sm_107f -mattr=+ptx94 -march=nvptx64 %s | FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefixes=FORMAT
; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mattr=+ptx94 -mcpu=sm_107f | %ptxas-verify -arch=sm_107f %}
; ---------------------------------------------------------------------------
@@ -8,6 +9,19 @@
define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
; kind=f16(0), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -71,6 +85,13 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta1(ptr addrspa
define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
; kind=f16(0), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -117,6 +138,13 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_cta2(ptr addrspa
define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
; kind=tf32(1), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -162,6 +190,12 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta1(ptr addrsp
define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
; kind=tf32(1), cta_group=2, collector_a=discard(0), collector_b=fill(2)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -205,6 +239,13 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_cta2(ptr addrsp
define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
; kind=f8f6f4(2), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -250,6 +291,12 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta1(ptr addr
define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
; kind=f8f6f4(2), cta_group=2, collector_a=discard(0), collector_b=use(3)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=f8f6f4 */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -293,6 +340,19 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f8f6f4_cta2(ptr addr
define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -356,6 +416,13 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrsp
define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -402,6 +469,12 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrsp
define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
; kind=f16(0), scale_d, cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 5, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 3, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 7, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 7, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 5, <4 x i32> %disable_output_lanev4, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -444,6 +517,12 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta1(ptr
define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4) {
; kind=tf32(1), scale_d, cta_group=1, collector_a=discard(0), collector_b=fill(2)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta1(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 4, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 6, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 2, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 2, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 4, <4 x i32> %disable_output_lanev4, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -486,6 +565,12 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta1(pt
define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
; kind=f16(0), scale_d, cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 5, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 3, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 7, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 7, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 5, <8 x i32> %disable_output_lanev8, /* kind=f16 */ i32 0, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -529,6 +614,12 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_f16_scale_d_cta2(ptr
define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8) {
; kind=tf32(1), scale_d, cta_group=2, collector_a=discard(0), collector_b=fill(2)
+; FORMAT-LABEL: define void @tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta2(
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 4, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 6, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 2, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.scale_d.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 2, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.scale_d.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 4, <8 x i32> %disable_output_lanev8, /* kind=tf32 */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_tf32_scale_d_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
>From c8e7d1cd86fa7791d078258c04e1cbae09d37a96 Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Wed, 12 Aug 2026 21:18:40 +0200
Subject: [PATCH 6/7] [NVPTX] More kind::ti16 test combinations
---
.../CodeGen/NVPTX/tcgen05-mma-collector-b.ll | 42 ++++++++++
...n05-mma-disable-output-lane-collector-b.ll | 12 +++
.../NVPTX/tcgen05-mma-sp-collector-b.ll | 56 +++++++++++++
...-mma-sp-disable-output-lane-collector-b.ll | 12 +++
.../CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll | 78 +++++++++++++++++++
5 files changed, 200 insertions(+)
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
index f688f0c2a9c03..54369a3cdef6c 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-collector-b.ll
@@ -352,6 +352,12 @@ define void @tcgen05_mma_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addr
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -381,6 +387,12 @@ define void @tcgen05_mma_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addr
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use.collector::b::fill [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::lastuse.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::lastuse [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 1)
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 2)
@@ -395,6 +407,12 @@ define void @tcgen05_mma_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr addr
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 2)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 0)
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 2, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 3, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 1, i32 1, i32 0)
ret void
}
@@ -410,6 +428,14 @@ define void @tcgen05_mma_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addr
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -432,6 +458,14 @@ define void @tcgen05_mma_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addr
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], %rd1, %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::lastuse.collector::b::lastuse [%r1], [%r3], %rd2, %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::lastuse [%r1], [%r3], %rd2, %r2, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 1)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 1)
@@ -439,6 +473,14 @@ define void @tcgen05_mma_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr addr
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 2)
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 0)
call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 4, i32 2, i32 1, i32 0)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
index 3fe4f0b904682..1e537fe368f83 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-disable-output-lane-collector-b.ll
@@ -355,6 +355,8 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=lastuse */ i32 1)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -384,6 +386,8 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::lastuse.collector::b::use [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill.collector::b::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2)
@@ -408,6 +412,10 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta1(ptr addrspace
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 3, i32 0)
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=discard(0)
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=use(3), tensor.ashift
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 1, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=fill(2), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %disable_output_lanev4, i32 4, i32 2, i32 1)
ret void
}
@@ -425,6 +433,7 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
; CHECK-LABEL: tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -450,6 +459,7 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 1)
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=fill(2)
@@ -464,6 +474,8 @@ define void @tcgen05_mma_disable_output_lane_collector_b_ti16_cta2(ptr addrspace
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 1, i32 0)
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0)
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=use(3), tensor
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 3)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
index c4e9a738ff9d9..dc3e5274719e6 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-collector-b.ll
@@ -416,6 +416,12 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr a
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=fill */ i32 2)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=fill */ i32 2, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 1, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -446,6 +452,12 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use.collector::b::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::lastuse.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: ret;
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=lastuse(1)
call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 1)
@@ -473,6 +485,18 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta1(ptr addrspace(6) %dtmem, ptr a
call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 3, i32 0)
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=discard(0)
call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 2)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=fill(2), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 2, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=use(3), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 3, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=lastuse(1), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 1)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=discard(0), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 1, i32 1, i32 0)
ret void
}
@@ -488,6 +512,14 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr a
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=lastuse */ i32 1)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, /* kind=ti16 */ i32 4, /* cta_group= */ i32 2, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -511,6 +543,14 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::lastuse.collector::b::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1;
; CHECK-NEXT: ret;
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=lastuse(1)
call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 1)
@@ -524,6 +564,22 @@ define void @tcgen05_mma_sp_collector_b_ti16_cta2(ptr addrspace(6) %dtmem, ptr a
call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0)
call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=fill(2)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 2)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=use(3)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 0, i32 3)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=lastuse(1), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 1)
+ ; kind=ti16(4), cta_group=2, collector_a=lastuse(1), collector_b=discard(0), ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i32 4, i32 2, i32 1, i32 0)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
index 476e5761d24b4..e8795d1a454f3 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-sp-disable-output-lane-collector-b.ll
@@ -353,6 +353,8 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrsp
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=lastuse */ i32 1)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=lastuse */ i32 1)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -383,6 +385,8 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrsp
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::lastuse.collector::b::use [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill.collector::b::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 1)
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=fill(2)
@@ -407,6 +411,10 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta1(ptr addrsp
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 3, i32 0)
; kind=ti16(4), cta_group=1, collector_a=discard(0), collector_b=discard(0)
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=1, collector_a=lastuse(1), collector_b=use(3), tensor.ashift
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 1, i32 3)
+ ; kind=ti16(4), cta_group=1, collector_a=fill(2), collector_b=lastuse(1)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %disable_output_lanev4, i32 4, i32 2, i32 1)
ret void
}
@@ -423,6 +431,7 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrsp
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=use */ i32 3)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=use */ i32 3)
; CHECK-LABEL: tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -448,6 +457,7 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrsp
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse.collector::b::use [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::discard.collector::b::use [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 1)
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=fill(2), tensor
@@ -460,6 +470,8 @@ define void @tcgen05_mma_sp_disable_output_lane_collector_b_ti16_cta2(ptr addrsp
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 1, i32 0)
; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=discard(0)
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 0)
+ ; kind=ti16(4), cta_group=2, collector_a=discard(0), collector_b=use(3), tensor
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %disable_output_lanev8, i32 4, i32 0, i32 3)
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
index 07d32eac8f0a4..b23363a29dedb 100644
--- a/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
+++ b/llvm/test/CodeGen/NVPTX/tcgen05-mma-ti16-kind.ll
@@ -278,6 +278,10 @@ define void @tcgen05_mma_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtmem,
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_ti16_disable_output_lane_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -300,12 +304,20 @@ define void @tcgen05_mma_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtmem,
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::lastuse [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::fill [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.collector::a::use [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::1.kind::ti16.ashift.collector::a::lastuse [%r1], [%r7], %rd2, %r2, {%r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 1, i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <4 x i32> %mask, i32 4, i32 1, i32 0)
ret void
}
@@ -320,6 +332,11 @@ define void @tcgen05_mma_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtmem,
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_ti16_disable_output_lane_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -343,12 +360,22 @@ define void @tcgen05_mma_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtmem,
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::lastuse [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::fill [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::use [%r1], %rd1, %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.collector::a::use [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
+; CHECK-NEXT: tcgen05.mma.cta_group::2.kind::ti16.ashift.collector::a::lastuse [%r1], [%r11], %rd2, %r2, {%r7, %r8, %r9, %r10, %r3, %r4, %r5, %r6}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 1, i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, <8 x i32> %mask, i32 4, i32 1, i32 0)
ret void
}
@@ -363,6 +390,10 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtme
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_ti16_disable_output_lane_cta1(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -386,12 +417,20 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta1(ptr addrspace(6) %dtme
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::discard [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::lastuse [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::fill [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.collector::a::use [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::1.kind::ti16.ashift.collector::a::lastuse [%r1], [%r8], %rd2, [%r3], %r2, {%r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 1, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg1(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg1.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <4 x i32> %mask, i32 4, i32 1, i32 0)
ret void
}
@@ -406,6 +445,11 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtme
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=discard */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=fill */ i32 2, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=use */ i32 3, /* collector_b=discard */ i32 0)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, /* kind=ti16 */ i32 4, /* collector_a=lastuse */ i32 1, /* collector_b=discard */ i32 0)
; CHECK-LABEL: tcgen05_mma_sp_ti16_disable_output_lane_cta2(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -430,12 +474,22 @@ define void @tcgen05_mma_sp_ti16_disable_output_lane_cta2(ptr addrspace(6) %dtme
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::discard [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::lastuse [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::fill [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::fill [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::use [%r1], %rd1, %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.collector::a::use [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
+; CHECK-NEXT: tcgen05.mma.sp.cta_group::2.kind::ti16.ashift.collector::a::lastuse [%r1], [%r12], %rd2, [%r3], %r2, {%r8, %r9, %r10, %r11, %r4, %r5, %r6, %r7}, %p1;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 1, i32 0)
call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 1, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 2, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.shared.disable_output_lane.cg2(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 3, i32 0)
+ call void @llvm.nvvm.tcgen05.mma.sp.tensor.disable_output_lane.cg2.ashift(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, <8 x i32> %mask, i32 4, i32 1, i32 0)
ret void
}
@@ -544,6 +598,10 @@ define void @tcgen05_mma_ws_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addr
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
; CHECK-LABEL: tcgen05_mma_ws_ti16_zero_col_mask(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -565,11 +623,19 @@ define void @tcgen05_mma_ws_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr addr
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r3], %rd2, %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r3], %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::fill [%r1], %rd1, %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::fill [%r1], [%r3], %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::use [%r1], %rd1, %rd2, %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.cta_group::1.kind::ti16.collector::b0::use [%r1], [%r3], %rd2, %r2, %p1, %rd3;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 1)
call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.ws.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i64 %zero_col_mask, i32 4, i32 0, i32 3)
ret void
}
@@ -583,6 +649,10 @@ define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr a
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=discard */ i32 0)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=lastuse */ i32 1)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=fill */ i32 2)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
+; FORMAT-NEXT: call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, /* kind=ti16 */ i32 4, /* collector_b_buffer=b0 */ i32 0, /* collector_b=use */ i32 3)
; CHECK-LABEL: tcgen05_mma_ws_sp_ti16_zero_col_mask(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<2>;
@@ -605,10 +675,18 @@ define void @tcgen05_mma_ws_sp_ti16_zero_col_mask(ptr addrspace(6) %dtmem, ptr a
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::discard [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::lastuse [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::fill [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::fill [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::use [%r1], %rd1, %rd2, [%r3], %r2, %p1, %rd3;
+; CHECK-NEXT: tcgen05.mma.ws.sp.cta_group::1.kind::ti16.collector::b0::use [%r1], [%r4], %rd2, [%r3], %r2, %p1, %rd3;
; CHECK-NEXT: ret;
call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 0)
call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 1)
call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 1)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 2)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.shared.zero_col_mask(ptr addrspace(6) %dtmem, i64 %ashared, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 3)
+ call void @llvm.nvvm.tcgen05.mma.ws.sp.tensor.zero_col_mask(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, ptr addrspace(6) %spmetadata, i64 %zero_col_mask, i32 4, i32 0, i32 3)
ret void
}
>From 61d69df4ea9de59e3b6afa9bf793e172bbb4ce8b Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Wed, 12 Aug 2026 23:55:02 +0200
Subject: [PATCH 7/7] [NVPTX] Fix wrong conflict resolution
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 4 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 51 -------------------------
2 files changed, 2 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index f20c3ed9fedf5..4c3c7792f277b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -174,9 +174,9 @@ def hasTcgen05ShiftSupport : PredOr<[SM100a, SM103a, SM110a]>;
def hasTcgen05MMAScaleInputDImm : PredOr<[SM100f]>;
def hasTcgen05MMAI8Kind : PredOr<[SM100a, SM110a]>;
-def hasTcgen05MMASparseMxf4 : PredOr<[SM100a, SM103a, SM110a]>;
+def hasTcgen05MMASparseMxf4 : PredOr<[SM100a, SM103a, SM110a, SM107a]>;
def hasTcgen05MMASparseMxf4nvf4
- : PredAnd<[PTX87, PredOr<[SM100a, SM103a, SM110a]>]>;
+ : PredAnd<[PTX87, PredOr<[SM100a, SM103a, SM110a, SM107a]>]>;
def hasTcgen05LdRedSupport : PredAnd<[PTX88, PredOr<[SM103f, SM110f]>]>;
def hasReduxSyncF32 : PredOr<[SM100f]>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index e1576e04f372f..b2f24c1d5d2b3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -151,57 +151,6 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
return hasAnyFeature({NVPTX::SM100f, NVPTX::SM110f});
}
- // Checks Rubin family extensions support.
- // - TMA S2G im2col_w mode support
- // - tcgen05.commit shared mem A variants.
- bool hasRubinFamilySupport() const { return hasAnyFeature({NVPTX::SM107f}); }
-
- // Checks tcgen05.shift instruction support.
- bool hasTcgen05ShiftSupport() const {
- return hasAnyFeature({NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a});
- }
-
- bool hasTcgen05MMAScaleInputDImm() const {
- return hasAnyFeature({NVPTX::SM100f});
- }
-
- bool hasTcgen05MMAI8Kind() const {
- return hasAnyFeature({NVPTX::SM100a, NVPTX::SM110a});
- }
-
- bool hasTcgen05MMASparseMxf4nvf4() const {
- return hasFeature(NVPTX::PTX87) &&
- hasAnyFeature(
- {NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a, NVPTX::SM107a});
- }
-
- bool hasTcgen05MMASparseMxf4() const {
- return hasAnyFeature(
- {NVPTX::SM100a, NVPTX::SM103a, NVPTX::SM110a, NVPTX::SM107a});
- }
-
- bool hasTcgen05LdRedSupport() const {
- return hasFeature(NVPTX::PTX88) &&
- hasAnyFeature({NVPTX::SM103f, NVPTX::SM110f});
- }
-
- bool hasReduxSyncF32() const { return hasAnyFeature({NVPTX::SM100f}); }
-
- bool hasMMABlockScale() const { return hasAnyFeature({NVPTX::SM120f}); }
-
- bool hasMMASparseBlockScaleF4() const {
- return hasAnyFeature({NVPTX::SM120a, NVPTX::SM121a});
- }
-
- bool hasMMAWithMXF4NVF4Scale4xE8M0() const {
- return hasFeature(NVPTX::PTX91) && hasAnyFeature({NVPTX::SM120f});
- }
-
- bool hasMMASparseWithMXF4NVF4Scale4xE8M0() const {
- return hasFeature(NVPTX::PTX91) &&
- hasAnyFeature({NVPTX::SM120a, NVPTX::SM121a});
- }
-
// f32x2 instructions in Blackwell family
bool hasF32x2Instructions() const;
More information about the llvm-commits
mailing list