[clang] [CIR][NVPTX] Support m16n16k16 WMMA builtins (PR #234812)

Aditya Tipre via cfe-commits cfe-commits at lists.llvm.org
Thu Oct 22 18:10:07 PDT 2026


https://github.com/atipre created https://github.com/llvm/llvm-project/pull/234812

Emit CIR for the ten SM70/PTX60 m16n16k16 floating-point WMMA builtins. Marshal FP16/FP32 fragments through NVVM load, store, and MMA intrinsics, preserving packed-half representations and selecting layout and saturation variants.

Add CIR and shared classic/CIR LLVM checks for all 44 supported combinations, plus target-feature and invalid-argument diagnostics.

Validation: All 36 relevant lit tests passed. SM70 PTX generation succeeded, and all 44 matrix instruction mnemonics matched classic codegen in order.

Related: https://github.com/llvm/llvm-project/issues/179278

Assisted-by: OpenAI Codex

>From a3207ba99201af39dad988517259bcd27568ba88 Mon Sep 17 00:00:00 2001
From: atipre <tipreaditya at gmail.com>
Date: Thu, 22 Oct 2026 21:03:51 -0400
Subject: [PATCH] [CIR][NVPTX] Support m16n16k16 WMMA builtins

Emit CIR for the ten SM70/PTX60 m16n16k16 floating-point WMMA builtins.
Marshal FP16/FP32 fragments through NVVM load, store, and MMA intrinsics,
preserving packed-half representations and selecting layout and
saturation variants.

Add CIR and shared classic/CIR LLVM checks for all 44 supported
combinations, plus target-feature and invalid-argument diagnostics.

Validation: 36 relevant lit tests pass. SM70 PTX generation succeeds,
with all 44 matrix instruction mnemonics matching classic codegen.

Related: https://github.com/llvm/llvm-project/issues/179278

Assisted-by: OpenAI Codex
---
 clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp  | 143 +++
 .../CodeGenCUDA/builtins-nvptx-wmma-errors.cu |  73 ++
 .../CIR/CodeGenCUDA/builtins-nvptx-wmma.cu    | 872 ++++++++++++++++++
 3 files changed, 1088 insertions(+)
 create mode 100644 clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma-errors.cu
 create mode 100644 clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma.cu

diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 146dad6b0ec413..5214245d6d3993 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -15,8 +15,11 @@
 #include "mlir/IR/Value.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/NVPTXAddrSpace.h"
 
+#include <algorithm>
+
 using namespace clang;
 using namespace clang::CIRGen;
 
@@ -130,6 +133,141 @@ static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf,
   return xchg.getResult();
 }
 
+// FP16 WMMA fragments are stored in i32 slots, but their intrinsic register
+// representation is <2 x half>. FP32 fragments use float in both
+// representations.
+static mlir::Type getWMMARegisterType(mlir::Type storageType) {
+  if (mlir::isa<cir::SingleType>(storageType))
+    return storageType;
+  return cir::VectorType::getFixed(cir::FP16Type::get(storageType.getContext()),
+                                   2);
+}
+
+static cir::StructType getWMMAResultType(CIRGenBuilderTy &builder,
+                                         mlir::Type registerType,
+                                         unsigned numRegisters) {
+  llvm::SmallVector<mlir::Type, 8> members(numRegisters, registerType);
+  return builder.getAnonRecordTy(members, /*packed=*/false,
+                                 cir::RecordType::getAllDataKinds(members));
+}
+
+static void loadWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
+                             Address src, unsigned numRegisters,
+                             llvm::SmallVectorImpl<mlir::Value> &values) {
+  mlir::Type registerType = getWMMARegisterType(src.getElementType());
+  // Every slot is four bytes, even when the fragment contains packed halves.
+  CharUnits alignment =
+      std::min(src.getAlignment(), CharUnits::fromQuantity(4));
+  for (unsigned i = 0; i < numRegisters; ++i) {
+    mlir::Value ptr = builder.createPtrStride(loc, src.emitRawPointer(),
+                                              builder.getSInt32(i, loc));
+    mlir::Value value =
+        builder.createAlignedLoad(loc, src.getElementType(), ptr, alignment);
+    values.push_back(builder.createBitcast(loc, value, registerType));
+  }
+}
+
+static void storeWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
+                              mlir::Value result, Address dst,
+                              unsigned numRegisters) {
+  CharUnits alignment =
+      std::min(dst.getAlignment(), CharUnits::fromQuantity(4));
+  for (unsigned i = 0; i < numRegisters; ++i) {
+    mlir::Value value = cir::ExtractMemberOp::create(builder, loc, result, i);
+    value = builder.createBitcast(loc, value, dst.getElementType());
+    mlir::Value ptr = builder.createPtrStride(loc, dst.emitRawPointer(),
+                                              builder.getSInt32(i, loc));
+    builder.createStore(loc, value, Address(ptr, alignment));
+  }
+}
+
+static std::optional<mlir::Value> emitWMMALoad(CIRGenFunction &cgf,
+                                               const CallExpr *expr,
+                                               llvm::StringRef fragment) {
+  auto &builder = cgf.getBuilder();
+  mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+  Address dst = cgf.emitPointerWithAlignment(expr->getArg(0));
+  mlir::Value src = cgf.emitScalarExpr(expr->getArg(1));
+  mlir::Value ldm = cgf.emitScalarExpr(expr->getArg(2));
+  auto isColMajor = expr->getArg(3)->getIntegerConstantExpr(cgf.getContext());
+  if (!isColMajor)
+    return std::nullopt;
+
+  bool isFloat = mlir::isa<cir::SingleType>(dst.getElementType());
+  unsigned numRegisters = fragment == "c" && !isFloat ? 4 : 8;
+  std::string name = ("nvvm.wmma.m16n16k16.load." + fragment +
+                      (*isColMajor != 0 ? ".col.stride." : ".row.stride.") +
+                      (isFloat ? "f32" : "f16"))
+                         .str();
+  mlir::Type resultType = getWMMAResultType(
+      builder, getWMMARegisterType(dst.getElementType()), numRegisters);
+  mlir::Value result = builder.emitIntrinsicCallOp(loc, name, resultType,
+                                                   mlir::ValueRange{src, ldm});
+  storeWMMAFragment(builder, loc, result, dst, numRegisters);
+  return mlir::Value{};
+}
+
+static std::optional<mlir::Value> emitWMMAStore(CIRGenFunction &cgf,
+                                                const CallExpr *expr) {
+  auto &builder = cgf.getBuilder();
+  mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+  mlir::Value dst = cgf.emitScalarExpr(expr->getArg(0));
+  Address src = cgf.emitPointerWithAlignment(expr->getArg(1));
+  mlir::Value ldm = cgf.emitScalarExpr(expr->getArg(2));
+  auto isColMajor = expr->getArg(3)->getIntegerConstantExpr(cgf.getContext());
+  if (!isColMajor)
+    return std::nullopt;
+
+  bool isFloat = mlir::isa<cir::SingleType>(src.getElementType());
+  std::string name = (llvm::Twine("nvvm.wmma.m16n16k16.store.d.") +
+                      (*isColMajor != 0 ? "col.stride." : "row.stride.") +
+                      (isFloat ? "f32" : "f16"))
+                         .str();
+  llvm::SmallVector<mlir::Value, 10> values{dst};
+  loadWMMAFragment(builder, loc, src, isFloat ? 8 : 4, values);
+  values.push_back(ldm);
+  cir::LLVMIntrinsicCallOp::create(builder, loc, builder.getStringAttr(name),
+                                   /*resType=*/mlir::Type{}, values);
+  return mlir::Value{};
+}
+
+static std::optional<mlir::Value> emitWMMAMma(CIRGenFunction &cgf,
+                                              const CallExpr *expr) {
+  auto &builder = cgf.getBuilder();
+  mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+  Address dst = cgf.emitPointerWithAlignment(expr->getArg(0));
+  Address srcA = cgf.emitPointerWithAlignment(expr->getArg(1));
+  Address srcB = cgf.emitPointerWithAlignment(expr->getArg(2));
+  Address srcC = cgf.emitPointerWithAlignment(expr->getArg(3));
+  auto layout = expr->getArg(4)->getIntegerConstantExpr(cgf.getContext());
+  auto satf = expr->getArg(5)->getIntegerConstantExpr(cgf.getContext());
+  if (!layout || layout->getSExtValue() < 0 || layout->getSExtValue() > 3 ||
+      !satf)
+    return std::nullopt;
+
+  bool isFloatC = mlir::isa<cir::SingleType>(srcC.getElementType());
+  bool isFloatD = mlir::isa<cir::SingleType>(dst.getElementType());
+  const char *layouts[] = {"row.row", "row.col", "col.row", "col.col"};
+  std::string name =
+      (llvm::Twine("nvvm.wmma.m16n16k16.mma.") +
+       layouts[layout->getSExtValue()] + (isFloatD ? ".f32" : ".f16") +
+       (isFloatC ? ".f32" : ".f16") + (*satf != 0 ? ".satfinite" : ""))
+          .str();
+
+  // Read every input before writing D, since the fragments may alias.
+  llvm::SmallVector<mlir::Value, 24> values;
+  loadWMMAFragment(builder, loc, srcA, 8, values);
+  loadWMMAFragment(builder, loc, srcB, 8, values);
+  loadWMMAFragment(builder, loc, srcC, isFloatC ? 8 : 4, values);
+  unsigned numResults = isFloatD ? 8 : 4;
+  mlir::Type resultType = getWMMAResultType(
+      builder, getWMMARegisterType(dst.getElementType()), numResults);
+  mlir::Value result =
+      builder.emitIntrinsicCallOp(loc, name, resultType, values);
+  storeWMMAFragment(builder, loc, result, dst, numResults);
+  return mlir::Value{};
+}
+
 std::optional<mlir::Value>
 CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
   switch (builtinId) {
@@ -414,9 +552,12 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
     return mlir::Value{};
   // FP MMA loads
   case NVPTX::BI__hmma_m16n16k16_ld_a:
+    return emitWMMALoad(*this, expr, "a");
   case NVPTX::BI__hmma_m16n16k16_ld_b:
+    return emitWMMALoad(*this, expr, "b");
   case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
   case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
+    return emitWMMALoad(*this, expr, "c");
   case NVPTX::BI__hmma_m32n8k16_ld_a:
   case NVPTX::BI__hmma_m32n8k16_ld_b:
   case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
@@ -482,6 +623,7 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
     return mlir::Value{};
   case NVPTX::BI__hmma_m16n16k16_st_c_f16:
   case NVPTX::BI__hmma_m16n16k16_st_c_f32:
+    return emitWMMAStore(*this, expr);
   case NVPTX::BI__hmma_m32n8k16_st_c_f16:
   case NVPTX::BI__hmma_m32n8k16_st_c_f32:
   case NVPTX::BI__hmma_m8n32k16_st_c_f16:
@@ -503,6 +645,7 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
   case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
   case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
   case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
+    return emitWMMAMma(*this, expr);
   case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
   case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
   case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma-errors.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma-errors.cu
new file mode 100644
index 00000000000000..48f1ae400f6ce2
--- /dev/null
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma-errors.cu
@@ -0,0 +1,73 @@
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_60 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-cir -DFEATURES -verify=feature %s -o /dev/null
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx42 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-cir -DFEATURES -verify=feature %s -o /dev/null
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-cir -DNEGATIVE_LAYOUT -verify=layout %s -o /dev/null
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-cir -DLARGE_LAYOUT -verify=layout %s -o /dev/null
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-cir -DCONSTANTS -verify=constant %s -o /dev/null
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device \
+// RUN:   -fsyntax-only -DCONSTANTS -verify=constant %s
+
+#define DEVICE extern "C" __attribute__((device))
+
+#ifdef FEATURES
+DEVICE void features(int *i, float *f, unsigned stride) {
+  // feature-error-re at +1 {{'__hmma_m16n16k16_ld_a' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_ld_a(i, i, stride, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_ld_b' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_ld_b(i, i, stride, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_ld_c_f16' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_ld_c_f16(i, i, stride, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_ld_c_f32' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_ld_c_f32(f, f, stride, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_st_c_f16' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_st_c_f16(i, i, stride, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_st_c_f32' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_st_c_f32(f, f, stride, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_mma_f16f16' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_mma_f16f16(i, i, i, i, 0, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_mma_f32f16' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_mma_f32f16(f, i, i, i, 0, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_mma_f16f32' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_mma_f16f32(i, i, i, f, 0, 0);
+  // feature-error-re at +1 {{'__hmma_m16n16k16_mma_f32f32' needs target feature {{.*}}sm_70{{.*}}ptx60}}
+  __hmma_m16n16k16_mma_f32f32(f, i, i, f, 0, 0);
+}
+#endif
+
+// Reject out-of-range compute layouts without indexing past the layout table.
+#ifdef NEGATIVE_LAYOUT
+DEVICE void negative_layout(int *p) {
+  // layout-error at +1 {{unimplemented builtin call: __hmma_m16n16k16_mma_f16f16}}
+  __hmma_m16n16k16_mma_f16f16(p, p, p, p, -1, 0);
+}
+#endif
+#ifdef LARGE_LAYOUT
+DEVICE void large_layout(int *p) {
+  // layout-error at +1 {{unimplemented builtin call: __hmma_m16n16k16_mma_f16f16}}
+  __hmma_m16n16k16_mma_f16f16(p, p, p, p, 4, 0);
+}
+#endif
+
+#ifdef CONSTANTS
+DEVICE void constant_arguments(int *p, int layout, int satfinite) {
+  // constant-error at +1 {{argument to '__hmma_m16n16k16_ld_a' must be a constant integer}}
+  __hmma_m16n16k16_ld_a(p, p, 16, layout);
+  // constant-error at +1 {{argument to '__hmma_m16n16k16_st_c_f16' must be a constant integer}}
+  __hmma_m16n16k16_st_c_f16(p, p, 16, layout);
+  // constant-error at +1 {{argument to '__hmma_m16n16k16_mma_f16f16' must be a constant integer}}
+  __hmma_m16n16k16_mma_f16f16(p, p, p, p, layout, 0);
+  // constant-error at +1 {{argument to '__hmma_m16n16k16_mma_f16f16' must be a constant integer}}
+  __hmma_m16n16k16_mma_f16f16(p, p, p, p, 0, satfinite);
+}
+#endif
+
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma.cu
new file mode 100644
index 00000000000000..51e69f7cbe5c38
--- /dev/null
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvptx-wmma.cu
@@ -0,0 +1,872 @@
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-cir %s -o - | FileCheck %s --check-prefix=CIR --enable-var-scope
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device -fclangir \
+// RUN:   -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM --enable-var-scope
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_70 \
+// RUN:   -target-feature +ptx60 -x cuda -fcuda-is-device \
+// RUN:   -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM --enable-var-scope
+
+// Cover the SM70/PTX60 m16n16k16 floating-point WMMA builtins. FP16
+// fragments use i32 storage slots and <2 x half> intrinsic registers.
+// The LLVM checks are shared with classic codegen.
+
+#define DEVICE extern "C" __attribute__((device))
+
+// CIR-LABEL: @load_a_f16_row(
+// LLVM-LABEL: @load_a_f16_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[SRC:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[STRIDE:.*]] = load i32, ptr %{{.*}}, align 4
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.a.row.stride.f16"
+// CIR: %[[E0:.*]] = cir.extract_member %[[RESULT]][0]
+// CIR: cir.cast bitcast %[[E0]] : !cir.vector<2 x !cir.f16> -> !s32i
+// CIR: cir.extract_member %[[RESULT]][7]
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p0(ptr %[[SRC]], i32 %[[STRIDE]])
+// LLVM: %[[E0:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 0
+// LLVM: %[[BITS0:.*]] = bitcast <2 x half> %[[E0]] to i32
+// LLVM: store i32 %[[BITS0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 1
+// LLVM: %[[BITS1:.*]] = bitcast <2 x half> %[[E1]] to i32
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store i32 %[[BITS1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 2
+// LLVM: %[[BITS2:.*]] = bitcast <2 x half> %[[E2]] to i32
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store i32 %[[BITS2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 3
+// LLVM: %[[BITS3:.*]] = bitcast <2 x half> %[[E3]] to i32
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store i32 %[[BITS3]], ptr %[[P3]], align 4
+// LLVM: %[[E4:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 4
+// LLVM: %[[BITS4:.*]] = bitcast <2 x half> %[[E4]] to i32
+// LLVM: %[[P4:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 4
+// LLVM: store i32 %[[BITS4]], ptr %[[P4]], align 4
+// LLVM: %[[E5:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 5
+// LLVM: %[[BITS5:.*]] = bitcast <2 x half> %[[E5]] to i32
+// LLVM: %[[P5:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 5
+// LLVM: store i32 %[[BITS5]], ptr %[[P5]], align 4
+// LLVM: %[[E6:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 6
+// LLVM: %[[BITS6:.*]] = bitcast <2 x half> %[[E6]] to i32
+// LLVM: %[[P6:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 6
+// LLVM: store i32 %[[BITS6]], ptr %[[P6]], align 4
+// LLVM: %[[E7:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 7
+// LLVM: %[[BITS7:.*]] = bitcast <2 x half> %[[E7]] to i32
+// LLVM: %[[P7:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 7
+// LLVM: store i32 %[[BITS7]], ptr %[[P7]], align 4
+// LLVM: ret void
+DEVICE void load_a_f16_row(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_ld_a(dst, src, stride, 0);
+}
+
+// CIR-LABEL: @load_a_f16_col(
+// LLVM-LABEL: @load_a_f16_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.a.col.stride.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p0(ptr %{{.*}}, i32 %{{.*}})
+DEVICE void load_a_f16_col(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_ld_a(dst, src, stride, 1);
+}
+
+// CIR-LABEL: @load_b_f16_row(
+// LLVM-LABEL: @load_b_f16_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[SRC:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[STRIDE:.*]] = load i32, ptr %{{.*}}, align 4
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.b.row.stride.f16"
+// CIR: %[[E0:.*]] = cir.extract_member %[[RESULT]][0]
+// CIR: cir.cast bitcast %[[E0]] : !cir.vector<2 x !cir.f16> -> !s32i
+// CIR: cir.extract_member %[[RESULT]][7]
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p0(ptr %[[SRC]], i32 %[[STRIDE]])
+// LLVM: %[[E0:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 0
+// LLVM: %[[BITS0:.*]] = bitcast <2 x half> %[[E0]] to i32
+// LLVM: store i32 %[[BITS0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 1
+// LLVM: %[[BITS1:.*]] = bitcast <2 x half> %[[E1]] to i32
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store i32 %[[BITS1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 2
+// LLVM: %[[BITS2:.*]] = bitcast <2 x half> %[[E2]] to i32
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store i32 %[[BITS2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 3
+// LLVM: %[[BITS3:.*]] = bitcast <2 x half> %[[E3]] to i32
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store i32 %[[BITS3]], ptr %[[P3]], align 4
+// LLVM: %[[E4:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 4
+// LLVM: %[[BITS4:.*]] = bitcast <2 x half> %[[E4]] to i32
+// LLVM: %[[P4:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 4
+// LLVM: store i32 %[[BITS4]], ptr %[[P4]], align 4
+// LLVM: %[[E5:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 5
+// LLVM: %[[BITS5:.*]] = bitcast <2 x half> %[[E5]] to i32
+// LLVM: %[[P5:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 5
+// LLVM: store i32 %[[BITS5]], ptr %[[P5]], align 4
+// LLVM: %[[E6:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 6
+// LLVM: %[[BITS6:.*]] = bitcast <2 x half> %[[E6]] to i32
+// LLVM: %[[P6:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 6
+// LLVM: store i32 %[[BITS6]], ptr %[[P6]], align 4
+// LLVM: %[[E7:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 7
+// LLVM: %[[BITS7:.*]] = bitcast <2 x half> %[[E7]] to i32
+// LLVM: %[[P7:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 7
+// LLVM: store i32 %[[BITS7]], ptr %[[P7]], align 4
+// LLVM: ret void
+DEVICE void load_b_f16_row(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_ld_b(dst, src, stride, 0);
+}
+
+// CIR-LABEL: @load_b_f16_col(
+// LLVM-LABEL: @load_b_f16_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.b.col.stride.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p0(ptr %{{.*}}, i32 %{{.*}})
+DEVICE void load_b_f16_col(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_ld_b(dst, src, stride, 1);
+}
+
+// CIR-LABEL: @load_c_f16_row(
+// LLVM-LABEL: @load_c_f16_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[SRC:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[STRIDE:.*]] = load i32, ptr %{{.*}}, align 4
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.c.row.stride.f16"
+// CIR: %[[E0:.*]] = cir.extract_member %[[RESULT]][0]
+// CIR: cir.cast bitcast %[[E0]] : !cir.vector<2 x !cir.f16> -> !s32i
+// CIR: cir.extract_member %[[RESULT]][3]
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p0(ptr %[[SRC]], i32 %[[STRIDE]])
+// LLVM: %[[E0:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 0
+// LLVM: %[[BITS0:.*]] = bitcast <2 x half> %[[E0]] to i32
+// LLVM: store i32 %[[BITS0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 1
+// LLVM: %[[BITS1:.*]] = bitcast <2 x half> %[[E1]] to i32
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store i32 %[[BITS1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 2
+// LLVM: %[[BITS2:.*]] = bitcast <2 x half> %[[E2]] to i32
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store i32 %[[BITS2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 3
+// LLVM: %[[BITS3:.*]] = bitcast <2 x half> %[[E3]] to i32
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store i32 %[[BITS3]], ptr %[[P3]], align 4
+// LLVM: ret void
+DEVICE void load_c_f16_row(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_ld_c_f16(dst, src, stride, 0);
+}
+
+// CIR-LABEL: @load_c_f16_col(
+// LLVM-LABEL: @load_c_f16_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.c.col.stride.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p0(ptr %{{.*}}, i32 %{{.*}})
+DEVICE void load_c_f16_col(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_ld_c_f16(dst, src, stride, 1);
+}
+
+// CIR-LABEL: @load_c_f32_row(
+// LLVM-LABEL: @load_c_f32_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[SRC:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[STRIDE:.*]] = load i32, ptr %{{.*}}, align 4
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.c.row.stride.f32"
+// CIR: %[[E0:.*]] = cir.extract_member %[[RESULT]][0]
+// CIR: cir.extract_member %[[RESULT]][7]
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p0(ptr %[[SRC]], i32 %[[STRIDE]])
+// LLVM: %[[E0:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 0
+// LLVM: store float %[[E0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 1
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store float %[[E1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 2
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store float %[[E2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 3
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store float %[[E3]], ptr %[[P3]], align 4
+// LLVM: %[[E4:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 4
+// LLVM: %[[P4:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 4
+// LLVM: store float %[[E4]], ptr %[[P4]], align 4
+// LLVM: %[[E5:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 5
+// LLVM: %[[P5:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 5
+// LLVM: store float %[[E5]], ptr %[[P5]], align 4
+// LLVM: %[[E6:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 6
+// LLVM: %[[P6:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 6
+// LLVM: store float %[[E6]], ptr %[[P6]], align 4
+// LLVM: %[[E7:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 7
+// LLVM: %[[P7:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 7
+// LLVM: store float %[[E7]], ptr %[[P7]], align 4
+// LLVM: ret void
+DEVICE void load_c_f32_row(float *dst, const float *src, unsigned stride) {
+  __hmma_m16n16k16_ld_c_f32(dst, src, stride, 0);
+}
+
+// CIR-LABEL: @load_c_f32_col(
+// LLVM-LABEL: @load_c_f32_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.load.c.col.stride.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p0(ptr %{{.*}}, i32 %{{.*}})
+DEVICE void load_c_f32_col(float *dst, const float *src, unsigned stride) {
+  __hmma_m16n16k16_ld_c_f32(dst, src, stride, 1);
+}
+
+// CIR-LABEL: @store_f16_row(
+// LLVM-LABEL: @store_f16_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[SRC:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[STRIDE:.*]] = load i32, ptr %{{.*}}, align 4
+// CIR: cir.cast bitcast {{.*}} : !s32i -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.store.d.row.stride.f16"
+// LLVM: %[[S0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[SR0:.*]] = bitcast i32 %[[S0]] to <2 x half>
+// LLVM: %[[SP1:.*]] = getelementptr{{.*}} i32, ptr %[[SRC]], i{{32|64}} 1
+// LLVM: %[[S1:.*]] = load i32, ptr %[[SP1]], align 4
+// LLVM: %[[SR1:.*]] = bitcast i32 %[[S1]] to <2 x half>
+// LLVM: %[[SP2:.*]] = getelementptr{{.*}} i32, ptr %[[SRC]], i{{32|64}} 2
+// LLVM: %[[S2:.*]] = load i32, ptr %[[SP2]], align 4
+// LLVM: %[[SR2:.*]] = bitcast i32 %[[S2]] to <2 x half>
+// LLVM: %[[SP3:.*]] = getelementptr{{.*}} i32, ptr %[[SRC]], i{{32|64}} 3
+// LLVM: %[[S3:.*]] = load i32, ptr %[[SP3]], align 4
+// LLVM: %[[SR3:.*]] = bitcast i32 %[[S3]] to <2 x half>
+// LLVM: call void @llvm.nvvm.wmma.m16n16k16.store.d.row.stride.f16.p0(ptr %[[DST]], <2 x half> %[[SR0]], <2 x half> %[[SR1]], <2 x half> %[[SR2]], <2 x half> %[[SR3]], i32 %[[STRIDE]])
+// LLVM: ret void
+DEVICE void store_f16_row(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_st_c_f16(dst, src, stride, 0);
+}
+
+// CIR-LABEL: @store_f16_col(
+// LLVM-LABEL: @store_f16_col(
+// CIR: cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.store.d.col.stride.f16"
+// LLVM: call void @llvm.nvvm.wmma.m16n16k16.store.d.col.stride.f16.p0(ptr %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, i32 %{{.*}})
+// LLVM: ret void
+DEVICE void store_f16_col(int *dst, const int *src, unsigned stride) {
+  __hmma_m16n16k16_st_c_f16(dst, src, stride, 1);
+}
+
+// CIR-LABEL: @store_f32_row(
+// LLVM-LABEL: @store_f32_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[SRC:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[STRIDE:.*]] = load i32, ptr %{{.*}}, align 4
+// CIR: cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.store.d.row.stride.f32"
+// LLVM: %[[S0:.*]] = load float, ptr %{{.*}}, align 4
+// LLVM: %[[SP1:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 1
+// LLVM: %[[S1:.*]] = load float, ptr %[[SP1]], align 4
+// LLVM: %[[SP2:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 2
+// LLVM: %[[S2:.*]] = load float, ptr %[[SP2]], align 4
+// LLVM: %[[SP3:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 3
+// LLVM: %[[S3:.*]] = load float, ptr %[[SP3]], align 4
+// LLVM: %[[SP4:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 4
+// LLVM: %[[S4:.*]] = load float, ptr %[[SP4]], align 4
+// LLVM: %[[SP5:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 5
+// LLVM: %[[S5:.*]] = load float, ptr %[[SP5]], align 4
+// LLVM: %[[SP6:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 6
+// LLVM: %[[S6:.*]] = load float, ptr %[[SP6]], align 4
+// LLVM: %[[SP7:.*]] = getelementptr{{.*}} float, ptr %[[SRC]], i{{32|64}} 7
+// LLVM: %[[S7:.*]] = load float, ptr %[[SP7]], align 4
+// LLVM: call void @llvm.nvvm.wmma.m16n16k16.store.d.row.stride.f32.p0(ptr %[[DST]], float %[[S0]], float %[[S1]], float %[[S2]], float %[[S3]], float %[[S4]], float %[[S5]], float %[[S6]], float %[[S7]], i32 %[[STRIDE]])
+// LLVM: ret void
+DEVICE void store_f32_row(float *dst, const float *src, unsigned stride) {
+  __hmma_m16n16k16_st_c_f32(dst, src, stride, 0);
+}
+
+// CIR-LABEL: @store_f32_col(
+// LLVM-LABEL: @store_f32_col(
+// CIR: cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.store.d.col.stride.f32"
+// LLVM: call void @llvm.nvvm.wmma.m16n16k16.store.d.col.stride.f32.p0(ptr %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, i32 %{{.*}})
+// LLVM: ret void
+DEVICE void store_f32_col(float *dst, const float *src, unsigned stride) {
+  __hmma_m16n16k16_st_c_f32(dst, src, stride, 1);
+}
+
+// CIR-LABEL: @mma_f16_f16_row_row(
+// LLVM-LABEL: @mma_f16_f16_row_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[A:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[B:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[C:.*]] = load ptr, ptr %{{.*}}
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f16.f16"
+// CIR: cir.extract_member %[[RESULT]][0]
+// CIR: cir.extract_member %[[RESULT]][3]
+// LLVM: %[[A0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[AR0:.*]] = bitcast i32 %[[A0]] to <2 x half>
+// LLVM: %[[AP1:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 1
+// LLVM: %[[A1:.*]] = load i32, ptr %[[AP1]], align 4
+// LLVM: %[[AR1:.*]] = bitcast i32 %[[A1]] to <2 x half>
+// LLVM: %[[AP2:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 2
+// LLVM: %[[A2:.*]] = load i32, ptr %[[AP2]], align 4
+// LLVM: %[[AR2:.*]] = bitcast i32 %[[A2]] to <2 x half>
+// LLVM: %[[AP3:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 3
+// LLVM: %[[A3:.*]] = load i32, ptr %[[AP3]], align 4
+// LLVM: %[[AR3:.*]] = bitcast i32 %[[A3]] to <2 x half>
+// LLVM: %[[AP4:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 4
+// LLVM: %[[A4:.*]] = load i32, ptr %[[AP4]], align 4
+// LLVM: %[[AR4:.*]] = bitcast i32 %[[A4]] to <2 x half>
+// LLVM: %[[AP5:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 5
+// LLVM: %[[A5:.*]] = load i32, ptr %[[AP5]], align 4
+// LLVM: %[[AR5:.*]] = bitcast i32 %[[A5]] to <2 x half>
+// LLVM: %[[AP6:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 6
+// LLVM: %[[A6:.*]] = load i32, ptr %[[AP6]], align 4
+// LLVM: %[[AR6:.*]] = bitcast i32 %[[A6]] to <2 x half>
+// LLVM: %[[AP7:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 7
+// LLVM: %[[A7:.*]] = load i32, ptr %[[AP7]], align 4
+// LLVM: %[[AR7:.*]] = bitcast i32 %[[A7]] to <2 x half>
+// LLVM: %[[B0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[BR0:.*]] = bitcast i32 %[[B0]] to <2 x half>
+// LLVM: %[[BP1:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 1
+// LLVM: %[[B1:.*]] = load i32, ptr %[[BP1]], align 4
+// LLVM: %[[BR1:.*]] = bitcast i32 %[[B1]] to <2 x half>
+// LLVM: %[[BP2:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 2
+// LLVM: %[[B2:.*]] = load i32, ptr %[[BP2]], align 4
+// LLVM: %[[BR2:.*]] = bitcast i32 %[[B2]] to <2 x half>
+// LLVM: %[[BP3:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 3
+// LLVM: %[[B3:.*]] = load i32, ptr %[[BP3]], align 4
+// LLVM: %[[BR3:.*]] = bitcast i32 %[[B3]] to <2 x half>
+// LLVM: %[[BP4:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 4
+// LLVM: %[[B4:.*]] = load i32, ptr %[[BP4]], align 4
+// LLVM: %[[BR4:.*]] = bitcast i32 %[[B4]] to <2 x half>
+// LLVM: %[[BP5:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 5
+// LLVM: %[[B5:.*]] = load i32, ptr %[[BP5]], align 4
+// LLVM: %[[BR5:.*]] = bitcast i32 %[[B5]] to <2 x half>
+// LLVM: %[[BP6:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 6
+// LLVM: %[[B6:.*]] = load i32, ptr %[[BP6]], align 4
+// LLVM: %[[BR6:.*]] = bitcast i32 %[[B6]] to <2 x half>
+// LLVM: %[[BP7:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 7
+// LLVM: %[[B7:.*]] = load i32, ptr %[[BP7]], align 4
+// LLVM: %[[BR7:.*]] = bitcast i32 %[[B7]] to <2 x half>
+// LLVM: %[[C0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[CR0:.*]] = bitcast i32 %[[C0]] to <2 x half>
+// LLVM: %[[CP1:.*]] = getelementptr{{.*}} i32, ptr %[[C]], i{{32|64}} 1
+// LLVM: %[[C1:.*]] = load i32, ptr %[[CP1]], align 4
+// LLVM: %[[CR1:.*]] = bitcast i32 %[[C1]] to <2 x half>
+// LLVM: %[[CP2:.*]] = getelementptr{{.*}} i32, ptr %[[C]], i{{32|64}} 2
+// LLVM: %[[C2:.*]] = load i32, ptr %[[CP2]], align 4
+// LLVM: %[[CR2:.*]] = bitcast i32 %[[C2]] to <2 x half>
+// LLVM: %[[CP3:.*]] = getelementptr{{.*}} i32, ptr %[[C]], i{{32|64}} 3
+// LLVM: %[[C3:.*]] = load i32, ptr %[[CP3]], align 4
+// LLVM: %[[CR3:.*]] = bitcast i32 %[[C3]] to <2 x half>
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f16.f16(<2 x half> %[[AR0]], <2 x half> %[[AR1]], <2 x half> %[[AR2]], <2 x half> %[[AR3]], <2 x half> %[[AR4]], <2 x half> %[[AR5]], <2 x half> %[[AR6]], <2 x half> %[[AR7]], <2 x half> %[[BR0]], <2 x half> %[[BR1]], <2 x half> %[[BR2]], <2 x half> %[[BR3]], <2 x half> %[[BR4]], <2 x half> %[[BR5]], <2 x half> %[[BR6]], <2 x half> %[[BR7]], <2 x half> %[[CR0]], <2 x half> %[[CR1]], <2 x half> %[[CR2]], <2 x half> %[[CR3]])
+// LLVM: %[[E0:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 0
+// LLVM: %[[BITS0:.*]] = bitcast <2 x half> %[[E0]] to i32
+// LLVM: store i32 %[[BITS0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 1
+// LLVM: %[[BITS1:.*]] = bitcast <2 x half> %[[E1]] to i32
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store i32 %[[BITS1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 2
+// LLVM: %[[BITS2:.*]] = bitcast <2 x half> %[[E2]] to i32
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store i32 %[[BITS2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 3
+// LLVM: %[[BITS3:.*]] = bitcast <2 x half> %[[E3]] to i32
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store i32 %[[BITS3]], ptr %[[P3]], align 4
+// LLVM: ret void
+DEVICE void mma_f16_f16_row_row(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 0, 0);
+}
+
+// CIR-LABEL: @mma_f16_f16_row_row_satfinite(
+// LLVM-LABEL: @mma_f16_f16_row_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f16.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f16.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_row_row_satfinite(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 0, 1);
+}
+
+// CIR-LABEL: @mma_f16_f16_row_col(
+// LLVM-LABEL: @mma_f16_f16_row_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f16.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f16.f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_row_col(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 1, 0);
+}
+
+// CIR-LABEL: @mma_f16_f16_row_col_satfinite(
+// LLVM-LABEL: @mma_f16_f16_row_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f16.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f16.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_row_col_satfinite(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 1, 1);
+}
+
+// CIR-LABEL: @mma_f16_f16_col_row(
+// LLVM-LABEL: @mma_f16_f16_col_row(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f16.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f16.f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_col_row(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 2, 0);
+}
+
+// CIR-LABEL: @mma_f16_f16_col_row_satfinite(
+// LLVM-LABEL: @mma_f16_f16_col_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f16.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f16.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_col_row_satfinite(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 2, 1);
+}
+
+// CIR-LABEL: @mma_f16_f16_col_col(
+// LLVM-LABEL: @mma_f16_f16_col_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f16.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f16.f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_col_col(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 3, 0);
+}
+
+// CIR-LABEL: @mma_f16_f16_col_col_satfinite(
+// LLVM-LABEL: @mma_f16_f16_col_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f16.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f16.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f16_f16_col_col_satfinite(int *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f16f16(dst, a, b, c, 3, 1);
+}
+
+// CIR-LABEL: @mma_f16_f32_row_row(
+// LLVM-LABEL: @mma_f16_f32_row_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[A:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[B:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[C:.*]] = load ptr, ptr %{{.*}}
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f16.f32"
+// CIR: cir.extract_member %[[RESULT]][0]
+// CIR: cir.extract_member %[[RESULT]][3]
+// LLVM: %[[A0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[AR0:.*]] = bitcast i32 %[[A0]] to <2 x half>
+// LLVM: %[[AP1:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 1
+// LLVM: %[[A1:.*]] = load i32, ptr %[[AP1]], align 4
+// LLVM: %[[AR1:.*]] = bitcast i32 %[[A1]] to <2 x half>
+// LLVM: %[[AP2:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 2
+// LLVM: %[[A2:.*]] = load i32, ptr %[[AP2]], align 4
+// LLVM: %[[AR2:.*]] = bitcast i32 %[[A2]] to <2 x half>
+// LLVM: %[[AP3:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 3
+// LLVM: %[[A3:.*]] = load i32, ptr %[[AP3]], align 4
+// LLVM: %[[AR3:.*]] = bitcast i32 %[[A3]] to <2 x half>
+// LLVM: %[[AP4:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 4
+// LLVM: %[[A4:.*]] = load i32, ptr %[[AP4]], align 4
+// LLVM: %[[AR4:.*]] = bitcast i32 %[[A4]] to <2 x half>
+// LLVM: %[[AP5:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 5
+// LLVM: %[[A5:.*]] = load i32, ptr %[[AP5]], align 4
+// LLVM: %[[AR5:.*]] = bitcast i32 %[[A5]] to <2 x half>
+// LLVM: %[[AP6:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 6
+// LLVM: %[[A6:.*]] = load i32, ptr %[[AP6]], align 4
+// LLVM: %[[AR6:.*]] = bitcast i32 %[[A6]] to <2 x half>
+// LLVM: %[[AP7:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 7
+// LLVM: %[[A7:.*]] = load i32, ptr %[[AP7]], align 4
+// LLVM: %[[AR7:.*]] = bitcast i32 %[[A7]] to <2 x half>
+// LLVM: %[[B0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[BR0:.*]] = bitcast i32 %[[B0]] to <2 x half>
+// LLVM: %[[BP1:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 1
+// LLVM: %[[B1:.*]] = load i32, ptr %[[BP1]], align 4
+// LLVM: %[[BR1:.*]] = bitcast i32 %[[B1]] to <2 x half>
+// LLVM: %[[BP2:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 2
+// LLVM: %[[B2:.*]] = load i32, ptr %[[BP2]], align 4
+// LLVM: %[[BR2:.*]] = bitcast i32 %[[B2]] to <2 x half>
+// LLVM: %[[BP3:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 3
+// LLVM: %[[B3:.*]] = load i32, ptr %[[BP3]], align 4
+// LLVM: %[[BR3:.*]] = bitcast i32 %[[B3]] to <2 x half>
+// LLVM: %[[BP4:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 4
+// LLVM: %[[B4:.*]] = load i32, ptr %[[BP4]], align 4
+// LLVM: %[[BR4:.*]] = bitcast i32 %[[B4]] to <2 x half>
+// LLVM: %[[BP5:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 5
+// LLVM: %[[B5:.*]] = load i32, ptr %[[BP5]], align 4
+// LLVM: %[[BR5:.*]] = bitcast i32 %[[B5]] to <2 x half>
+// LLVM: %[[BP6:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 6
+// LLVM: %[[B6:.*]] = load i32, ptr %[[BP6]], align 4
+// LLVM: %[[BR6:.*]] = bitcast i32 %[[B6]] to <2 x half>
+// LLVM: %[[BP7:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 7
+// LLVM: %[[B7:.*]] = load i32, ptr %[[BP7]], align 4
+// LLVM: %[[BR7:.*]] = bitcast i32 %[[B7]] to <2 x half>
+// LLVM: %[[C0:.*]] = load float, ptr %{{.*}}, align 4
+// LLVM: %[[CP1:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 1
+// LLVM: %[[C1:.*]] = load float, ptr %[[CP1]], align 4
+// LLVM: %[[CP2:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 2
+// LLVM: %[[C2:.*]] = load float, ptr %[[CP2]], align 4
+// LLVM: %[[CP3:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 3
+// LLVM: %[[C3:.*]] = load float, ptr %[[CP3]], align 4
+// LLVM: %[[CP4:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 4
+// LLVM: %[[C4:.*]] = load float, ptr %[[CP4]], align 4
+// LLVM: %[[CP5:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 5
+// LLVM: %[[C5:.*]] = load float, ptr %[[CP5]], align 4
+// LLVM: %[[CP6:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 6
+// LLVM: %[[C6:.*]] = load float, ptr %[[CP6]], align 4
+// LLVM: %[[CP7:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 7
+// LLVM: %[[C7:.*]] = load float, ptr %[[CP7]], align 4
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f16.f32(<2 x half> %[[AR0]], <2 x half> %[[AR1]], <2 x half> %[[AR2]], <2 x half> %[[AR3]], <2 x half> %[[AR4]], <2 x half> %[[AR5]], <2 x half> %[[AR6]], <2 x half> %[[AR7]], <2 x half> %[[BR0]], <2 x half> %[[BR1]], <2 x half> %[[BR2]], <2 x half> %[[BR3]], <2 x half> %[[BR4]], <2 x half> %[[BR5]], <2 x half> %[[BR6]], <2 x half> %[[BR7]], float %[[C0]], float %[[C1]], float %[[C2]], float %[[C3]], float %[[C4]], float %[[C5]], float %[[C6]], float %[[C7]])
+// LLVM: %[[E0:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 0
+// LLVM: %[[BITS0:.*]] = bitcast <2 x half> %[[E0]] to i32
+// LLVM: store i32 %[[BITS0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 1
+// LLVM: %[[BITS1:.*]] = bitcast <2 x half> %[[E1]] to i32
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store i32 %[[BITS1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 2
+// LLVM: %[[BITS2:.*]] = bitcast <2 x half> %[[E2]] to i32
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store i32 %[[BITS2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { <2 x half>, <2 x half>, <2 x half>, <2 x half> } %[[RESULT]], 3
+// LLVM: %[[BITS3:.*]] = bitcast <2 x half> %[[E3]] to i32
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} i32, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store i32 %[[BITS3]], ptr %[[P3]], align 4
+// LLVM: ret void
+DEVICE void mma_f16_f32_row_row(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 0, 0);
+}
+
+// CIR-LABEL: @mma_f16_f32_row_row_satfinite(
+// LLVM-LABEL: @mma_f16_f32_row_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f16.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f16.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_row_row_satfinite(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 0, 1);
+}
+
+// CIR-LABEL: @mma_f16_f32_row_col(
+// LLVM-LABEL: @mma_f16_f32_row_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f16.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f16.f32(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_row_col(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 1, 0);
+}
+
+// CIR-LABEL: @mma_f16_f32_row_col_satfinite(
+// LLVM-LABEL: @mma_f16_f32_row_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f16.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f16.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_row_col_satfinite(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 1, 1);
+}
+
+// CIR-LABEL: @mma_f16_f32_col_row(
+// LLVM-LABEL: @mma_f16_f32_col_row(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f16.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f16.f32(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_col_row(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 2, 0);
+}
+
+// CIR-LABEL: @mma_f16_f32_col_row_satfinite(
+// LLVM-LABEL: @mma_f16_f32_col_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f16.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f16.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_col_row_satfinite(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 2, 1);
+}
+
+// CIR-LABEL: @mma_f16_f32_col_col(
+// LLVM-LABEL: @mma_f16_f32_col_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f16.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f16.f32(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_col_col(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 3, 0);
+}
+
+// CIR-LABEL: @mma_f16_f32_col_col_satfinite(
+// LLVM-LABEL: @mma_f16_f32_col_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f16.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { <2 x half>, <2 x half>, <2 x half>, <2 x half> } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f16.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f16_f32_col_col_satfinite(int *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f16f32(dst, a, b, c, 3, 1);
+}
+
+// CIR-LABEL: @mma_f32_f16_row_row(
+// LLVM-LABEL: @mma_f32_f16_row_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[A:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[B:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[C:.*]] = load ptr, ptr %{{.*}}
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f32.f16"
+// CIR: cir.extract_member %[[RESULT]][0]
+// CIR: cir.extract_member %[[RESULT]][7]
+// LLVM: %[[A0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[AR0:.*]] = bitcast i32 %[[A0]] to <2 x half>
+// LLVM: %[[AP1:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 1
+// LLVM: %[[A1:.*]] = load i32, ptr %[[AP1]], align 4
+// LLVM: %[[AR1:.*]] = bitcast i32 %[[A1]] to <2 x half>
+// LLVM: %[[AP2:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 2
+// LLVM: %[[A2:.*]] = load i32, ptr %[[AP2]], align 4
+// LLVM: %[[AR2:.*]] = bitcast i32 %[[A2]] to <2 x half>
+// LLVM: %[[AP3:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 3
+// LLVM: %[[A3:.*]] = load i32, ptr %[[AP3]], align 4
+// LLVM: %[[AR3:.*]] = bitcast i32 %[[A3]] to <2 x half>
+// LLVM: %[[AP4:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 4
+// LLVM: %[[A4:.*]] = load i32, ptr %[[AP4]], align 4
+// LLVM: %[[AR4:.*]] = bitcast i32 %[[A4]] to <2 x half>
+// LLVM: %[[AP5:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 5
+// LLVM: %[[A5:.*]] = load i32, ptr %[[AP5]], align 4
+// LLVM: %[[AR5:.*]] = bitcast i32 %[[A5]] to <2 x half>
+// LLVM: %[[AP6:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 6
+// LLVM: %[[A6:.*]] = load i32, ptr %[[AP6]], align 4
+// LLVM: %[[AR6:.*]] = bitcast i32 %[[A6]] to <2 x half>
+// LLVM: %[[AP7:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 7
+// LLVM: %[[A7:.*]] = load i32, ptr %[[AP7]], align 4
+// LLVM: %[[AR7:.*]] = bitcast i32 %[[A7]] to <2 x half>
+// LLVM: %[[B0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[BR0:.*]] = bitcast i32 %[[B0]] to <2 x half>
+// LLVM: %[[BP1:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 1
+// LLVM: %[[B1:.*]] = load i32, ptr %[[BP1]], align 4
+// LLVM: %[[BR1:.*]] = bitcast i32 %[[B1]] to <2 x half>
+// LLVM: %[[BP2:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 2
+// LLVM: %[[B2:.*]] = load i32, ptr %[[BP2]], align 4
+// LLVM: %[[BR2:.*]] = bitcast i32 %[[B2]] to <2 x half>
+// LLVM: %[[BP3:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 3
+// LLVM: %[[B3:.*]] = load i32, ptr %[[BP3]], align 4
+// LLVM: %[[BR3:.*]] = bitcast i32 %[[B3]] to <2 x half>
+// LLVM: %[[BP4:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 4
+// LLVM: %[[B4:.*]] = load i32, ptr %[[BP4]], align 4
+// LLVM: %[[BR4:.*]] = bitcast i32 %[[B4]] to <2 x half>
+// LLVM: %[[BP5:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 5
+// LLVM: %[[B5:.*]] = load i32, ptr %[[BP5]], align 4
+// LLVM: %[[BR5:.*]] = bitcast i32 %[[B5]] to <2 x half>
+// LLVM: %[[BP6:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 6
+// LLVM: %[[B6:.*]] = load i32, ptr %[[BP6]], align 4
+// LLVM: %[[BR6:.*]] = bitcast i32 %[[B6]] to <2 x half>
+// LLVM: %[[BP7:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 7
+// LLVM: %[[B7:.*]] = load i32, ptr %[[BP7]], align 4
+// LLVM: %[[BR7:.*]] = bitcast i32 %[[B7]] to <2 x half>
+// LLVM: %[[C0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[CR0:.*]] = bitcast i32 %[[C0]] to <2 x half>
+// LLVM: %[[CP1:.*]] = getelementptr{{.*}} i32, ptr %[[C]], i{{32|64}} 1
+// LLVM: %[[C1:.*]] = load i32, ptr %[[CP1]], align 4
+// LLVM: %[[CR1:.*]] = bitcast i32 %[[C1]] to <2 x half>
+// LLVM: %[[CP2:.*]] = getelementptr{{.*}} i32, ptr %[[C]], i{{32|64}} 2
+// LLVM: %[[C2:.*]] = load i32, ptr %[[CP2]], align 4
+// LLVM: %[[CR2:.*]] = bitcast i32 %[[C2]] to <2 x half>
+// LLVM: %[[CP3:.*]] = getelementptr{{.*}} i32, ptr %[[C]], i{{32|64}} 3
+// LLVM: %[[C3:.*]] = load i32, ptr %[[CP3]], align 4
+// LLVM: %[[CR3:.*]] = bitcast i32 %[[C3]] to <2 x half>
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f32.f16(<2 x half> %[[AR0]], <2 x half> %[[AR1]], <2 x half> %[[AR2]], <2 x half> %[[AR3]], <2 x half> %[[AR4]], <2 x half> %[[AR5]], <2 x half> %[[AR6]], <2 x half> %[[AR7]], <2 x half> %[[BR0]], <2 x half> %[[BR1]], <2 x half> %[[BR2]], <2 x half> %[[BR3]], <2 x half> %[[BR4]], <2 x half> %[[BR5]], <2 x half> %[[BR6]], <2 x half> %[[BR7]], <2 x half> %[[CR0]], <2 x half> %[[CR1]], <2 x half> %[[CR2]], <2 x half> %[[CR3]])
+// LLVM: %[[E0:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 0
+// LLVM: store float %[[E0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 1
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store float %[[E1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 2
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store float %[[E2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 3
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store float %[[E3]], ptr %[[P3]], align 4
+// LLVM: %[[E4:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 4
+// LLVM: %[[P4:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 4
+// LLVM: store float %[[E4]], ptr %[[P4]], align 4
+// LLVM: %[[E5:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 5
+// LLVM: %[[P5:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 5
+// LLVM: store float %[[E5]], ptr %[[P5]], align 4
+// LLVM: %[[E6:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 6
+// LLVM: %[[P6:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 6
+// LLVM: store float %[[E6]], ptr %[[P6]], align 4
+// LLVM: %[[E7:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 7
+// LLVM: %[[P7:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 7
+// LLVM: store float %[[E7]], ptr %[[P7]], align 4
+// LLVM: ret void
+DEVICE void mma_f32_f16_row_row(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 0, 0);
+}
+
+// CIR-LABEL: @mma_f32_f16_row_row_satfinite(
+// LLVM-LABEL: @mma_f32_f16_row_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f32.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f32.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_row_row_satfinite(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 0, 1);
+}
+
+// CIR-LABEL: @mma_f32_f16_row_col(
+// LLVM-LABEL: @mma_f32_f16_row_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f32.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f32.f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_row_col(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 1, 0);
+}
+
+// CIR-LABEL: @mma_f32_f16_row_col_satfinite(
+// LLVM-LABEL: @mma_f32_f16_row_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f32.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f32.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_row_col_satfinite(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 1, 1);
+}
+
+// CIR-LABEL: @mma_f32_f16_col_row(
+// LLVM-LABEL: @mma_f32_f16_col_row(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f32.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f32.f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_col_row(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 2, 0);
+}
+
+// CIR-LABEL: @mma_f32_f16_col_row_satfinite(
+// LLVM-LABEL: @mma_f32_f16_col_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f32.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f32.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_col_row_satfinite(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 2, 1);
+}
+
+// CIR-LABEL: @mma_f32_f16_col_col(
+// LLVM-LABEL: @mma_f32_f16_col_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f32.f16"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f32.f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_col_col(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 3, 0);
+}
+
+// CIR-LABEL: @mma_f32_f16_col_col_satfinite(
+// LLVM-LABEL: @mma_f32_f16_col_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f32.f16.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f32.f16.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+DEVICE void mma_f32_f16_col_col_satfinite(float *dst, const int *a, const int *b, const int *c) {
+  __hmma_m16n16k16_mma_f32f16(dst, a, b, c, 3, 1);
+}
+
+// CIR-LABEL: @mma_f32_f32_row_row(
+// LLVM-LABEL: @mma_f32_f32_row_row(
+// LLVM: %[[DST:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[A:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[B:.*]] = load ptr, ptr %{{.*}}
+// LLVM: %[[C:.*]] = load ptr, ptr %{{.*}}
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f32.f32"
+// CIR: cir.extract_member %[[RESULT]][0]
+// CIR: cir.extract_member %[[RESULT]][7]
+// LLVM: %[[A0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[AR0:.*]] = bitcast i32 %[[A0]] to <2 x half>
+// LLVM: %[[AP1:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 1
+// LLVM: %[[A1:.*]] = load i32, ptr %[[AP1]], align 4
+// LLVM: %[[AR1:.*]] = bitcast i32 %[[A1]] to <2 x half>
+// LLVM: %[[AP2:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 2
+// LLVM: %[[A2:.*]] = load i32, ptr %[[AP2]], align 4
+// LLVM: %[[AR2:.*]] = bitcast i32 %[[A2]] to <2 x half>
+// LLVM: %[[AP3:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 3
+// LLVM: %[[A3:.*]] = load i32, ptr %[[AP3]], align 4
+// LLVM: %[[AR3:.*]] = bitcast i32 %[[A3]] to <2 x half>
+// LLVM: %[[AP4:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 4
+// LLVM: %[[A4:.*]] = load i32, ptr %[[AP4]], align 4
+// LLVM: %[[AR4:.*]] = bitcast i32 %[[A4]] to <2 x half>
+// LLVM: %[[AP5:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 5
+// LLVM: %[[A5:.*]] = load i32, ptr %[[AP5]], align 4
+// LLVM: %[[AR5:.*]] = bitcast i32 %[[A5]] to <2 x half>
+// LLVM: %[[AP6:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 6
+// LLVM: %[[A6:.*]] = load i32, ptr %[[AP6]], align 4
+// LLVM: %[[AR6:.*]] = bitcast i32 %[[A6]] to <2 x half>
+// LLVM: %[[AP7:.*]] = getelementptr{{.*}} i32, ptr %[[A]], i{{32|64}} 7
+// LLVM: %[[A7:.*]] = load i32, ptr %[[AP7]], align 4
+// LLVM: %[[AR7:.*]] = bitcast i32 %[[A7]] to <2 x half>
+// LLVM: %[[B0:.*]] = load i32, ptr %{{.*}}, align 4
+// LLVM: %[[BR0:.*]] = bitcast i32 %[[B0]] to <2 x half>
+// LLVM: %[[BP1:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 1
+// LLVM: %[[B1:.*]] = load i32, ptr %[[BP1]], align 4
+// LLVM: %[[BR1:.*]] = bitcast i32 %[[B1]] to <2 x half>
+// LLVM: %[[BP2:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 2
+// LLVM: %[[B2:.*]] = load i32, ptr %[[BP2]], align 4
+// LLVM: %[[BR2:.*]] = bitcast i32 %[[B2]] to <2 x half>
+// LLVM: %[[BP3:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 3
+// LLVM: %[[B3:.*]] = load i32, ptr %[[BP3]], align 4
+// LLVM: %[[BR3:.*]] = bitcast i32 %[[B3]] to <2 x half>
+// LLVM: %[[BP4:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 4
+// LLVM: %[[B4:.*]] = load i32, ptr %[[BP4]], align 4
+// LLVM: %[[BR4:.*]] = bitcast i32 %[[B4]] to <2 x half>
+// LLVM: %[[BP5:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 5
+// LLVM: %[[B5:.*]] = load i32, ptr %[[BP5]], align 4
+// LLVM: %[[BR5:.*]] = bitcast i32 %[[B5]] to <2 x half>
+// LLVM: %[[BP6:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 6
+// LLVM: %[[B6:.*]] = load i32, ptr %[[BP6]], align 4
+// LLVM: %[[BR6:.*]] = bitcast i32 %[[B6]] to <2 x half>
+// LLVM: %[[BP7:.*]] = getelementptr{{.*}} i32, ptr %[[B]], i{{32|64}} 7
+// LLVM: %[[B7:.*]] = load i32, ptr %[[BP7]], align 4
+// LLVM: %[[BR7:.*]] = bitcast i32 %[[B7]] to <2 x half>
+// LLVM: %[[C0:.*]] = load float, ptr %{{.*}}, align 4
+// LLVM: %[[CP1:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 1
+// LLVM: %[[C1:.*]] = load float, ptr %[[CP1]], align 4
+// LLVM: %[[CP2:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 2
+// LLVM: %[[C2:.*]] = load float, ptr %[[CP2]], align 4
+// LLVM: %[[CP3:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 3
+// LLVM: %[[C3:.*]] = load float, ptr %[[CP3]], align 4
+// LLVM: %[[CP4:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 4
+// LLVM: %[[C4:.*]] = load float, ptr %[[CP4]], align 4
+// LLVM: %[[CP5:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 5
+// LLVM: %[[C5:.*]] = load float, ptr %[[CP5]], align 4
+// LLVM: %[[CP6:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 6
+// LLVM: %[[C6:.*]] = load float, ptr %[[CP6]], align 4
+// LLVM: %[[CP7:.*]] = getelementptr{{.*}} float, ptr %[[C]], i{{32|64}} 7
+// LLVM: %[[C7:.*]] = load float, ptr %[[CP7]], align 4
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f32.f32(<2 x half> %[[AR0]], <2 x half> %[[AR1]], <2 x half> %[[AR2]], <2 x half> %[[AR3]], <2 x half> %[[AR4]], <2 x half> %[[AR5]], <2 x half> %[[AR6]], <2 x half> %[[AR7]], <2 x half> %[[BR0]], <2 x half> %[[BR1]], <2 x half> %[[BR2]], <2 x half> %[[BR3]], <2 x half> %[[BR4]], <2 x half> %[[BR5]], <2 x half> %[[BR6]], <2 x half> %[[BR7]], float %[[C0]], float %[[C1]], float %[[C2]], float %[[C3]], float %[[C4]], float %[[C5]], float %[[C6]], float %[[C7]])
+// LLVM: %[[E0:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 0
+// LLVM: store float %[[E0]], ptr %[[P0:.*]], align 4
+// LLVM: %[[E1:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 1
+// LLVM: %[[P1:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 1
+// LLVM: store float %[[E1]], ptr %[[P1]], align 4
+// LLVM: %[[E2:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 2
+// LLVM: %[[P2:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 2
+// LLVM: store float %[[E2]], ptr %[[P2]], align 4
+// LLVM: %[[E3:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 3
+// LLVM: %[[P3:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 3
+// LLVM: store float %[[E3]], ptr %[[P3]], align 4
+// LLVM: %[[E4:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 4
+// LLVM: %[[P4:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 4
+// LLVM: store float %[[E4]], ptr %[[P4]], align 4
+// LLVM: %[[E5:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 5
+// LLVM: %[[P5:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 5
+// LLVM: store float %[[E5]], ptr %[[P5]], align 4
+// LLVM: %[[E6:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 6
+// LLVM: %[[P6:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 6
+// LLVM: store float %[[E6]], ptr %[[P6]], align 4
+// LLVM: %[[E7:.*]] = extractvalue { float, float, float, float, float, float, float, float } %[[RESULT]], 7
+// LLVM: %[[P7:.*]] = getelementptr{{.*}} float, ptr %[[DST]], i{{32|64}} 7
+// LLVM: store float %[[E7]], ptr %[[P7]], align 4
+// LLVM: ret void
+DEVICE void mma_f32_f32_row_row(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 0, 0);
+}
+
+// CIR-LABEL: @mma_f32_f32_row_row_satfinite(
+// LLVM-LABEL: @mma_f32_f32_row_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.row.f32.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.row.f32.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_row_row_satfinite(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 0, 1);
+}
+
+// CIR-LABEL: @mma_f32_f32_row_col(
+// LLVM-LABEL: @mma_f32_f32_row_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f32.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f32.f32(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_row_col(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 1, 0);
+}
+
+// CIR-LABEL: @mma_f32_f32_row_col_satfinite(
+// LLVM-LABEL: @mma_f32_f32_row_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.row.col.f32.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.row.col.f32.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_row_col_satfinite(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 1, 1);
+}
+
+// CIR-LABEL: @mma_f32_f32_col_row(
+// LLVM-LABEL: @mma_f32_f32_col_row(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f32.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f32.f32(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_col_row(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 2, 0);
+}
+
+// CIR-LABEL: @mma_f32_f32_col_row_satfinite(
+// LLVM-LABEL: @mma_f32_f32_col_row_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.row.f32.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.row.f32.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_col_row_satfinite(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 2, 1);
+}
+
+// CIR-LABEL: @mma_f32_f32_col_col(
+// LLVM-LABEL: @mma_f32_f32_col_col(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f32.f32"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f32.f32(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_col_col(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 3, 0);
+}
+
+// CIR-LABEL: @mma_f32_f32_col_col_satfinite(
+// LLVM-LABEL: @mma_f32_f32_col_col_satfinite(
+// CIR: %[[RESULT:.*]] = cir.call_llvm_intrinsic "nvvm.wmma.m16n16k16.mma.col.col.f32.f32.satfinite"
+// LLVM: %[[RESULT:.*]] = call{{.*}} { float, float, float, float, float, float, float, float } @llvm.nvvm.wmma.m16n16k16.mma.col.col.f32.f32.satfinite(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+DEVICE void mma_f32_f32_col_col_satfinite(float *dst, const int *a, const int *b, const float *c) {
+  __hmma_m16n16k16_mma_f32f32(dst, a, b, c, 3, 1);
+}
+



More information about the cfe-commits mailing list