[clang] [lldb] [llvm] [OpenCL][Clang] Add support for cooperative matrix extension (PR #221328)

via cfe-commits cfe-commits at lists.llvm.org
Fri Nov 13 14:50:18 PST 2026


================
@@ -4663,6 +4759,253 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Result);
   }
 
+  case Builtin::BIcoop_mat_load: {
+    auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+    assert(PtrTy && "arg0 must be of pointer type");
+    (void)PtrTy;
+    Address SrcAddr = EmitPointerWithAlignment(E->getArg(0));
+    EmitNonNullArgCheck(SrcAddr, E->getArg(0)->getType(),
+                        E->getArg(0)->getExprLoc(), FD, 0);
+    const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+    if (!MTy)
+      CGM.ErrorUnsupported(E, "coop_mat_load without coop_mat output operand");
+
+    auto Ptr = EmitScalarExpr(E->getArg(0));
+    auto Layout = EmitScalarExpr(E->getArg(1));
+    Layout = Builder.CreateZExtOrTrunc(Layout, Int32Ty);
+    auto Stride = EmitScalarExpr(E->getArg(2));
+    Stride = Builder.CreateZExtOrTrunc(Stride, Int32Ty);
+    // Create a TargetExtType to represent the coop matrix type
+    llvm::TargetExtType *RetType = getTargetExtType(CGM, MTy);
+
+    // Set function type.
+    llvm::FunctionType *FTy = llvm::FunctionType::get(
+        RetType, {Ptr->getType(), Int32Ty, Int32Ty}, false);
+    // Function name mangling.
+    std::string Name =
+        getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+        getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+        getCoopMatFullSuffix(ConvertType(E->getType()));
+
+    llvm::FunctionCallee LoadFn =
+        CGM.getModule().getOrInsertFunction(Name, FTy);
+    if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
+      F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+    auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
+    NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+    return RValue::get(NewCall);
+  }
+
+  case Builtin::BIcoop_mat_store: {
+    auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+    assert(PtrTy && "arg0 must be of pointer type");
+    (void)PtrTy;
+    Address Dst = EmitPointerWithAlignment(E->getArg(0));
+    EmitNonNullArgCheck(Dst, E->getArg(0)->getType(),
+                        E->getArg(0)->getExprLoc(), FD, 0);
+    const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+    if (!MTy)
+      CGM.ErrorUnsupported(E, "coop_mat_store without coop_mat input operand");
+
+    auto Ptr = EmitScalarExpr(E->getArg(0));
+    auto Arg0 = EmitScalarExpr(E->getArg(1));
+    auto Layout = EmitScalarExpr(E->getArg(2));
+    Layout = Builder.CreateZExtOrTrunc(Layout, Int32Ty);
+    auto Stride = EmitScalarExpr(E->getArg(3));
+    Stride = Builder.CreateZExtOrTrunc(Stride, Int32Ty);
+    // Create a TargetExtType to represent the coop matrix type
+    llvm::TargetExtType *ArgType = getTargetExtType(CGM, MTy);
+    // Set function type.
+    llvm::FunctionType *FTy = llvm::FunctionType::get(
+        VoidTy, {Ptr->getType(), ArgType, Int32Ty, Int32Ty}, false);
+    // Function name mangling.
+    std::string Name =
+        getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+        getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+        getCoopMatFullSuffix(ConvertType(E->getArg(1)->getType()));
+
+    llvm::FunctionCallee StoreFn =
+        CGM.getModule().getOrInsertFunction(Name, FTy);
+    if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
+      F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+    auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
+    NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+    return RValue::get(NewCall);
+  }
+
+  case Builtin::BIcoop_mat_mulAdd: {
+    const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+    if (!MTy)
+      CGM.ErrorUnsupported(E,
+                           "coop_mat_mulAdd without coop_mat output operand");
+
+    auto Arg0 = E->getArg(0);
+    auto Arg1 = E->getArg(1);
+    auto Arg2 = E->getArg(2);
+    Value *Zero = Constant::getNullValue(Int32Ty);
+    auto MA = EmitScalarExpr(Arg0);
+    auto MB = EmitScalarExpr(Arg1);
+    auto MC = EmitScalarExpr(Arg2);
+    auto MatrixOperand =
+        (E->getNumArgs() == 4) ? EmitScalarExpr(E->getArg(3)) : Zero;
----------------
asudarsa-qti wrote:

Updated in 82659e7505cd2e2da8c14ebb2412b41bfbd33137

Thanks

https://github.com/llvm/llvm-project/pull/221328


More information about the cfe-commits mailing list