[clang] [CIR][NVPTX] Support m16n16k16 WMMA builtins (PR #234812)
Andy Kaylor via cfe-commits
cfe-commits at lists.llvm.org
Thu Nov 5 16:51:06 PST 2026
================
@@ -148,6 +153,270 @@ static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf,
return xchg.getResult();
}
+// Mirrors the WMMA load mappings in classic CodeGen.
+struct NVPTXMmaLdstInfo {
+ unsigned NumResults; // Number of elements to load/store
+ // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
+ unsigned IID_col;
+ unsigned IID_row;
+};
+
+#define MMA_INTR(geom_op_type, layout) \
+ llvm::Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
+#define MMA_LDST(n, geom_op_type) \
+ {n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row)}
+
+static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
+ switch (BuiltinID) {
+ // FP MMA loads
+ case NVPTX::BI__hmma_m16n16k16_ld_a:
+ return MMA_LDST(8, m16n16k16_load_a_f16);
+ case NVPTX::BI__hmma_m16n16k16_ld_b:
+ return MMA_LDST(8, m16n16k16_load_b_f16);
+ case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
+ return MMA_LDST(4, m16n16k16_load_c_f16);
+ case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
+ return MMA_LDST(8, m16n16k16_load_c_f32);
+ case NVPTX::BI__hmma_m32n8k16_ld_a:
+ return MMA_LDST(8, m32n8k16_load_a_f16);
+ case NVPTX::BI__hmma_m32n8k16_ld_b:
+ return MMA_LDST(8, m32n8k16_load_b_f16);
+ case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
+ return MMA_LDST(4, m32n8k16_load_c_f16);
+ case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
+ return MMA_LDST(8, m32n8k16_load_c_f32);
+ case NVPTX::BI__hmma_m8n32k16_ld_a:
+ return MMA_LDST(8, m8n32k16_load_a_f16);
+ case NVPTX::BI__hmma_m8n32k16_ld_b:
+ return MMA_LDST(8, m8n32k16_load_b_f16);
+ case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
+ return MMA_LDST(4, m8n32k16_load_c_f16);
+ case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
+ return MMA_LDST(8, m8n32k16_load_c_f32);
+
+ // Integer MMA loads
+ case NVPTX::BI__imma_m16n16k16_ld_a_s8:
+ return MMA_LDST(2, m16n16k16_load_a_s8);
+ case NVPTX::BI__imma_m16n16k16_ld_a_u8:
+ return MMA_LDST(2, m16n16k16_load_a_u8);
+ case NVPTX::BI__imma_m16n16k16_ld_b_s8:
+ return MMA_LDST(2, m16n16k16_load_b_s8);
+ case NVPTX::BI__imma_m16n16k16_ld_b_u8:
+ return MMA_LDST(2, m16n16k16_load_b_u8);
+ case NVPTX::BI__imma_m16n16k16_ld_c:
+ return MMA_LDST(8, m16n16k16_load_c_s32);
+ case NVPTX::BI__imma_m32n8k16_ld_a_s8:
+ return MMA_LDST(4, m32n8k16_load_a_s8);
+ case NVPTX::BI__imma_m32n8k16_ld_a_u8:
+ return MMA_LDST(4, m32n8k16_load_a_u8);
+ case NVPTX::BI__imma_m32n8k16_ld_b_s8:
+ return MMA_LDST(1, m32n8k16_load_b_s8);
+ case NVPTX::BI__imma_m32n8k16_ld_b_u8:
+ return MMA_LDST(1, m32n8k16_load_b_u8);
+ case NVPTX::BI__imma_m32n8k16_ld_c:
+ return MMA_LDST(8, m32n8k16_load_c_s32);
+ case NVPTX::BI__imma_m8n32k16_ld_a_s8:
+ return MMA_LDST(1, m8n32k16_load_a_s8);
+ case NVPTX::BI__imma_m8n32k16_ld_a_u8:
+ return MMA_LDST(1, m8n32k16_load_a_u8);
+ case NVPTX::BI__imma_m8n32k16_ld_b_s8:
+ return MMA_LDST(4, m8n32k16_load_b_s8);
+ case NVPTX::BI__imma_m8n32k16_ld_b_u8:
+ return MMA_LDST(4, m8n32k16_load_b_u8);
+ case NVPTX::BI__imma_m8n32k16_ld_c:
+ return MMA_LDST(8, m8n32k16_load_c_s32);
+
+ // Sub-integer MMA loads.
+ // Only row/col layout is supported by A/B fragments.
+ case NVPTX::BI__imma_m8n8k32_ld_a_s4:
+ return {1, 0, MMA_INTR(m8n8k32_load_a_s4, row)};
+ case NVPTX::BI__imma_m8n8k32_ld_a_u4:
+ return {1, 0, MMA_INTR(m8n8k32_load_a_u4, row)};
+ case NVPTX::BI__imma_m8n8k32_ld_b_s4:
+ return {1, MMA_INTR(m8n8k32_load_b_s4, col), 0};
+ case NVPTX::BI__imma_m8n8k32_ld_b_u4:
+ return {1, MMA_INTR(m8n8k32_load_b_u4, col), 0};
+ case NVPTX::BI__imma_m8n8k32_ld_c:
+ return MMA_LDST(2, m8n8k32_load_c_s32);
+ case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
+ return {1, 0, MMA_INTR(m8n8k128_load_a_b1, row)};
+ case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
+ return {1, MMA_INTR(m8n8k128_load_b_b1, col), 0};
+ case NVPTX::BI__bmma_m8n8k128_ld_c:
+ return MMA_LDST(2, m8n8k128_load_c_s32);
+
+ // Double MMA loads
+ case NVPTX::BI__dmma_m8n8k4_ld_a:
+ return MMA_LDST(1, m8n8k4_load_a_f64);
+ case NVPTX::BI__dmma_m8n8k4_ld_b:
+ return MMA_LDST(1, m8n8k4_load_b_f64);
+ case NVPTX::BI__dmma_m8n8k4_ld_c:
+ return MMA_LDST(2, m8n8k4_load_c_f64);
+
+ // Alternate float MMA loads
+ case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
+ return MMA_LDST(4, m16n16k16_load_a_bf16);
+ case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
+ return MMA_LDST(4, m16n16k16_load_b_bf16);
+ case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
+ return MMA_LDST(2, m8n32k16_load_a_bf16);
+ case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
+ return MMA_LDST(8, m8n32k16_load_b_bf16);
+ case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
+ return MMA_LDST(8, m32n8k16_load_a_bf16);
+ case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
+ return MMA_LDST(2, m32n8k16_load_b_bf16);
+ case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
+ return MMA_LDST(4, m16n16k8_load_a_tf32);
+ case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
+ return MMA_LDST(4, m16n16k8_load_b_tf32);
+ case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
+ return MMA_LDST(8, m16n16k8_load_c_f32);
+
+ default:
+ llvm_unreachable("Unknown WMMA load builtin");
+ }
+}
+#undef MMA_LDST
+#undef MMA_INTR
+
+// FP16 WMMA fragments are stored in i32 slots, but their intrinsic register
+// representation is <2 x half>. FP32 fragments use float in both
+// representations.
+static mlir::Type getWMMARegisterType(mlir::Type storageType) {
+ if (mlir::isa<cir::SingleType>(storageType))
+ return storageType;
+ return cir::VectorType::getFixed(cir::FP16Type::get(storageType.getContext()),
+ 2);
+}
+
+static cir::StructType getWMMAResultType(CIRGenBuilderTy &builder,
+ mlir::Type registerType,
+ unsigned numRegisters) {
+ llvm::SmallVector<mlir::Type, 8> members(numRegisters, registerType);
+ return builder.getAnonRecordTy(members, /*packed=*/false,
+ cir::RecordType::getAllDataKinds(members));
+}
+
+static void loadWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
+ Address src, unsigned numRegisters,
+ llvm::SmallVectorImpl<mlir::Value> &values) {
+ mlir::Type registerType = getWMMARegisterType(src.getElementType());
+ // Every slot is four bytes, even when the fragment contains packed halves.
+ CharUnits alignment =
+ std::min(src.getAlignment(), CharUnits::fromQuantity(4));
+ for (unsigned i = 0; i < numRegisters; ++i) {
+ mlir::Value ptr = builder.createPtrStride(loc, src.emitRawPointer(),
+ builder.getSInt32(i, loc));
+ mlir::Value value =
+ builder.createAlignedLoad(loc, src.getElementType(), ptr, alignment);
+ values.push_back(builder.createBitcast(loc, value, registerType));
+ }
+}
+
+static void storeWMMAFragment(CIRGenBuilderTy &builder, mlir::Location loc,
+ mlir::Value result, Address dst,
+ unsigned numRegisters) {
+ CharUnits alignment =
+ std::min(dst.getAlignment(), CharUnits::fromQuantity(4));
+ for (unsigned i = 0; i < numRegisters; ++i) {
+ mlir::Value value = result;
+ if (numRegisters != 1)
+ value = cir::ExtractMemberOp::create(builder, loc, result, i);
+ value = builder.createBitcast(loc, value, dst.getElementType());
+ mlir::Value ptr = builder.createPtrStride(loc, dst.emitRawPointer(),
+ builder.getSInt32(i, loc));
+ builder.createStore(loc, value, Address(ptr, alignment));
+ }
+}
+
+static std::optional<mlir::Value>
+emitWMMALoad(CIRGenFunction &cgf, const CallExpr *expr, unsigned builtinId) {
+ auto &builder = cgf.getBuilder();
+ mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+ Address dst = cgf.emitPointerWithAlignment(expr->getArg(0));
+ mlir::Value src = cgf.emitScalarExpr(expr->getArg(1));
+ mlir::Value ldm = cgf.emitScalarExpr(expr->getArg(2));
+ auto isColMajor = expr->getArg(3)->getIntegerConstantExpr(cgf.getContext());
----------------
andykaylor wrote:
Don't use `auto` here. Also, I'd prefer to see the explicit conversion from `std::optional<llvm::APSInt>` to a bool value as it is in classic codegen. It's more readable and a bit more efficient.
https://github.com/llvm/llvm-project/pull/234812
More information about the cfe-commits
mailing list