[clang] [CIR][NVPTX] Support m16n16k16 WMMA builtins (PR #234812)

Andy Kaylor via cfe-commits cfe-commits at lists.llvm.org
Thu Nov 5 16:51:06 PST 2026


================
@@ -148,6 +153,270 @@ static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf,
   return xchg.getResult();
 }
 
+// Mirrors the WMMA load mappings in classic CodeGen.
+struct NVPTXMmaLdstInfo {
+  unsigned NumResults; // Number of elements to load/store
+  // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
+  unsigned IID_col;
+  unsigned IID_row;
+};
+
+#define MMA_INTR(geom_op_type, layout)                                         \
+  llvm::Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
+#define MMA_LDST(n, geom_op_type)                                              \
+  {n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row)}
+
+static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
+  switch (BuiltinID) {
+  // FP MMA loads
+  case NVPTX::BI__hmma_m16n16k16_ld_a:
+    return MMA_LDST(8, m16n16k16_load_a_f16);
+  case NVPTX::BI__hmma_m16n16k16_ld_b:
+    return MMA_LDST(8, m16n16k16_load_b_f16);
+  case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
+    return MMA_LDST(4, m16n16k16_load_c_f16);
+  case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
+    return MMA_LDST(8, m16n16k16_load_c_f32);
+  case NVPTX::BI__hmma_m32n8k16_ld_a:
+    return MMA_LDST(8, m32n8k16_load_a_f16);
+  case NVPTX::BI__hmma_m32n8k16_ld_b:
+    return MMA_LDST(8, m32n8k16_load_b_f16);
+  case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
+    return MMA_LDST(4, m32n8k16_load_c_f16);
+  case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
+    return MMA_LDST(8, m32n8k16_load_c_f32);
+  case NVPTX::BI__hmma_m8n32k16_ld_a:
+    return MMA_LDST(8, m8n32k16_load_a_f16);
+  case NVPTX::BI__hmma_m8n32k16_ld_b:
+    return MMA_LDST(8, m8n32k16_load_b_f16);
+  case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
+    return MMA_LDST(4, m8n32k16_load_c_f16);
+  case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
+    return MMA_LDST(8, m8n32k16_load_c_f32);
+
+  // Integer MMA loads
+  case NVPTX::BI__imma_m16n16k16_ld_a_s8:
+    return MMA_LDST(2, m16n16k16_load_a_s8);
+  case NVPTX::BI__imma_m16n16k16_ld_a_u8:
+    return MMA_LDST(2, m16n16k16_load_a_u8);
+  case NVPTX::BI__imma_m16n16k16_ld_b_s8:
+    return MMA_LDST(2, m16n16k16_load_b_s8);
+  case NVPTX::BI__imma_m16n16k16_ld_b_u8:
+    return MMA_LDST(2, m16n16k16_load_b_u8);
+  case NVPTX::BI__imma_m16n16k16_ld_c:
+    return MMA_LDST(8, m16n16k16_load_c_s32);
+  case NVPTX::BI__imma_m32n8k16_ld_a_s8:
+    return MMA_LDST(4, m32n8k16_load_a_s8);
+  case NVPTX::BI__imma_m32n8k16_ld_a_u8:
+    return MMA_LDST(4, m32n8k16_load_a_u8);
+  case NVPTX::BI__imma_m32n8k16_ld_b_s8:
+    return MMA_LDST(1, m32n8k16_load_b_s8);
+  case NVPTX::BI__imma_m32n8k16_ld_b_u8:
+    return MMA_LDST(1, m32n8k16_load_b_u8);
+  case NVPTX::BI__imma_m32n8k16_ld_c:
+    return MMA_LDST(8, m32n8k16_load_c_s32);
+  case NVPTX::BI__imma_m8n32k16_ld_a_s8:
+    return MMA_LDST(1, m8n32k16_load_a_s8);
+  case NVPTX::BI__imma_m8n32k16_ld_a_u8:
+    return MMA_LDST(1, m8n32k16_load_a_u8);
+  case NVPTX::BI__imma_m8n32k16_ld_b_s8:
+    return MMA_LDST(4, m8n32k16_load_b_s8);
+  case NVPTX::BI__imma_m8n32k16_ld_b_u8:
+    return MMA_LDST(4, m8n32k16_load_b_u8);
+  case NVPTX::BI__imma_m8n32k16_ld_c:
+    return MMA_LDST(8, m8n32k16_load_c_s32);
+
+  // Sub-integer MMA loads.
+  // Only row/col layout is supported by A/B fragments.
+  case NVPTX::BI__imma_m8n8k32_ld_a_s4:
+    return {1, 0, MMA_INTR(m8n8k32_load_a_s4, row)};
+  case NVPTX::BI__imma_m8n8k32_ld_a_u4:
+    return {1, 0, MMA_INTR(m8n8k32_load_a_u4, row)};
+  case NVPTX::BI__imma_m8n8k32_ld_b_s4:
+    return {1, MMA_INTR(m8n8k32_load_b_s4, col), 0};
+  case NVPTX::BI__imma_m8n8k32_ld_b_u4:
+    return {1, MMA_INTR(m8n8k32_load_b_u4, col), 0};
+  case NVPTX::BI__imma_m8n8k32_ld_c:
+    return MMA_LDST(2, m8n8k32_load_c_s32);
+  case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
+    return {1, 0, MMA_INTR(m8n8k128_load_a_b1, row)};
+  case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
+    return {1, MMA_INTR(m8n8k128_load_b_b1, col), 0};
+  case NVPTX::BI__bmma_m8n8k128_ld_c:
+    return MMA_LDST(2, m8n8k128_load_c_s32);
+
+  // Double MMA loads
+  case NVPTX::BI__dmma_m8n8k4_ld_a:
+    return MMA_LDST(1, m8n8k4_load_a_f64);
+  case NVPTX::BI__dmma_m8n8k4_ld_b:
+    return MMA_LDST(1, m8n8k4_load_b_f64);
+  case NVPTX::BI__dmma_m8n8k4_ld_c:
+    return MMA_LDST(2, m8n8k4_load_c_f64);
+
+  // Alternate float MMA loads
+  case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
+    return MMA_LDST(4, m16n16k16_load_a_bf16);
+  case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
+    return MMA_LDST(4, m16n16k16_load_b_bf16);
+  case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
+    return MMA_LDST(2, m8n32k16_load_a_bf16);
+  case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
+    return MMA_LDST(8, m8n32k16_load_b_bf16);
+  case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
+    return MMA_LDST(8, m32n8k16_load_a_bf16);
+  case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
+    return MMA_LDST(2, m32n8k16_load_b_bf16);
+  case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
+    return MMA_LDST(4, m16n16k8_load_a_tf32);
+  case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
+    return MMA_LDST(4, m16n16k8_load_b_tf32);
+  case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
+    return MMA_LDST(8, m16n16k8_load_c_f32);
+
+  default:
+    llvm_unreachable("Unknown WMMA load builtin");
+  }
+}
+#undef MMA_LDST
+#undef MMA_INTR
+
+// FP16 WMMA fragments are stored in i32 slots, but their intrinsic register
+// representation is <2 x half>. FP32 fragments use float in both
+// representations.
+static mlir::Type getWMMARegisterType(mlir::Type storageType) {
+  if (mlir::isa<cir::SingleType>(storageType))
+    return storageType;
+  return cir::VectorType::getFixed(cir::FP16Type::get(storageType.getContext()),
+                                   2);
+}
+
+static cir::StructType getWMMAResultType(CIRGenBuilderTy &builder,
+                                         mlir::Type registerType,
+                                         unsigned numRegisters) {
+  llvm::SmallVector<mlir::Type, 8> members(numRegisters, registerType);
+  return builder.getAnonRecordTy(members, /*packed=*/false,
+                                 cir::RecordType::getAllDataKinds(members));
+}
+
+static void loadWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
+                             Address src, unsigned numRegisters,
+                             llvm::SmallVectorImpl<mlir::Value> &values) {
+  mlir::Type registerType = getWMMARegisterType(src.getElementType());
+  // Every slot is four bytes, even when the fragment contains packed halves.
+  CharUnits alignment =
+      std::min(src.getAlignment(), CharUnits::fromQuantity(4));
+  for (unsigned i = 0; i < numRegisters; ++i) {
+    mlir::Value ptr = builder.createPtrStride(loc, src.emitRawPointer(),
+                                              builder.getSInt32(i, loc));
+    mlir::Value value =
+        builder.createAlignedLoad(loc, src.getElementType(), ptr, alignment);
+    values.push_back(builder.createBitcast(loc, value, registerType));
+  }
+}
+
+static void storeWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
+                              mlir::Value result, Address dst,
+                              unsigned numRegisters) {
+  CharUnits alignment =
+      std::min(dst.getAlignment(), CharUnits::fromQuantity(4));
+  for (unsigned i = 0; i < numRegisters; ++i) {
+    mlir::Value value = result;
+    if (numRegisters != 1)
+      value = cir::ExtractMemberOp::create(builder, loc, result, i);
+    value = builder.createBitcast(loc, value, dst.getElementType());
+    mlir::Value ptr = builder.createPtrStride(loc, dst.emitRawPointer(),
+                                              builder.getSInt32(i, loc));
+    builder.createStore(loc, value, Address(ptr, alignment));
+  }
+}
+
+static std::optional<mlir::Value>
+emitWMMALoad(CIRGenFunction &cgf, const CallExpr *expr, unsigned builtinId) {
+  auto &builder = cgf.getBuilder();
+  mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+  Address dst = cgf.emitPointerWithAlignment(expr->getArg(0));
+  mlir::Value src = cgf.emitScalarExpr(expr->getArg(1));
+  mlir::Value ldm = cgf.emitScalarExpr(expr->getArg(2));
+  auto isColMajor = expr->getArg(3)->getIntegerConstantExpr(cgf.getContext());
----------------
andykaylor wrote:

Don't use `auto` here. Also, I'd prefer to see the explicit conversion from `std::optional<llvm::APSInt>` to a bool value as it is in classic codegen. It's more readable and a bit more efficient.

https://github.com/llvm/llvm-project/pull/234812


More information about the cfe-commits mailing list