[clang] [lldb] [llvm] [OpenCL][Clang] Add support for cooperative matrix extension (PR #221328)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 7 12:51:50 PDT 2026
https://github.com/asudarsa-qti updated https://github.com/llvm/llvm-project/pull/221328
>From a42d95ac8648cc09bec6be1d426b08f3de6de90a Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:14:42 -0400
Subject: [PATCH 01/38] [OpenCL][CoopMat] Patch 1 - Introduce Cooperative
matrix type
---
clang/include/clang/AST/ASTContext.h | 10 +++
clang/include/clang/AST/RecursiveASTVisitor.h | 11 +++
clang/include/clang/AST/TypeBase.h | 90 +++++++++++++++++++
clang/include/clang/AST/TypeLoc.h | 34 ++++++-
clang/include/clang/AST/TypeProperties.td | 13 +++
clang/include/clang/Basic/TypeNodes.td | 1 +
.../clang/Serialization/TypeBitCodes.def | 1 +
clang/lib/AST/ASTContext.cpp | 82 +++++++++++++++++
clang/lib/AST/Type.cpp | 38 ++++++++
clang/lib/AST/TypePrinter.cpp | 15 ++++
10 files changed, 294 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index b04b6335616615..539123df01298f 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -264,6 +264,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
mutable llvm::ContextualFoldingSet<DependentVectorType, ASTContext &>
DependentVectorTypes;
mutable llvm::FoldingSet<ConstantMatrixType> MatrixTypes;
+ mutable llvm::FoldingSet<CooperativeMatrixType> CooperativeMatrixTypes;
mutable llvm::ContextualFoldingSet<DependentSizedMatrixType, ASTContext &>
DependentSizedMatrixTypes;
mutable llvm::FoldingSet<FunctionNoProtoType> FunctionNoProtoTypes;
@@ -1917,6 +1918,15 @@ class ASTContext : public RefCountedBase<ASTContext> {
QualType ElementType, unsigned NumRows, unsigned NumColumns,
std::optional<MatrixType::LayoutKind> Layout = std::nullopt) const;
+ /// Return the unique reference to the cooperative matrix type of the
+ /// specified element type and size.
+ ///
+ /// \pre \p ElementType must be a valid matrix element type (see
+ /// MatrixType::isValidElementType).
+ QualType getCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use) const;
+
/// Return the unique reference to the matrix type of the specified element
/// type and size
QualType getDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index f144c890f6ee65..791536d03f8f42 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -1089,6 +1089,9 @@ DEF_TRAVERSE_TYPE(ExtVectorType, { TRY_TO(TraverseType(T->getElementType())); })
DEF_TRAVERSE_TYPE(ConstantMatrixType,
{ TRY_TO(TraverseType(T->getElementType())); })
+DEF_TRAVERSE_TYPE(CooperativeMatrixType,
+ { TRY_TO(TraverseType(T->getElementType())); })
+
DEF_TRAVERSE_TYPE(DependentSizedMatrixType, {
if (T->getRowExpr())
TRY_TO(TraverseStmt(T->getRowExpr()));
@@ -1409,6 +1412,14 @@ DEF_TRAVERSE_TYPELOC(ConstantMatrixType, {
TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
})
+DEF_TRAVERSE_TYPELOC(CooperativeMatrixType, {
+ TRY_TO(TraverseStmt(TL.getAttrScopeOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrUseOperand()));
+ TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
+})
+
DEF_TRAVERSE_TYPELOC(DependentSizedMatrixType, {
TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 6373794bb69f54..9b1c9ef646f1f5 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -2708,6 +2708,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
bool isSubscriptableVectorType() const;
bool isMatrixType() const; // Matrix type.
bool isConstantMatrixType() const; // Constant matrix type.
+ bool isCooperativeMatrixType() const; // Cooperative matrix type.
bool isOverflowBehaviorType() const; // Overflow behavior type.
bool isDependentAddressSpaceType() const; // value-dependent address space qualifier
bool isObjCObjectPointerType() const; // pointer to ObjC object
@@ -4482,6 +4483,7 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
+ T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4577,6 +4579,90 @@ class ConstantMatrixType final : public MatrixType {
}
};
+/// Represents a cooperative matrix type.
+class CooperativeMatrixType final : public MatrixType {
+protected:
+ friend class ASTContext;
+
+ /// Number of rows and columns.
+ unsigned NumRows;
+ unsigned NumColumns;
+
+ /// Scope and use
+ unsigned Scope;
+ unsigned Use;
+
+ static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
+
+ CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
+ unsigned NRows, unsigned NColumns, unsigned Use,
+ QualType CanonElementType);
+
+ CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
+ unsigned Scope, unsigned NRows, unsigned NColumns,
+ unsigned Use, QualType CanonElementType);
+
+public:
+ /// Returns the number of rows in the matrix.
+ unsigned getNumRows() const { return NumRows; }
+
+ /// Returns the number of columns in the matrix.
+ unsigned getNumColumns() const { return NumColumns; }
+
+ /// Returns the scope of the matrix.
+ unsigned getScope() const { return Scope; }
+
+ /// Returns the use of the matrix.
+ unsigned getUse() const { return Use; }
+
+ /// Returns the number of elements required to embed the matrix into a vector.
+ unsigned getNumElementsFlattened() const {
+ return getNumRows() * getNumColumns();
+ }
+
+ /// Returns true if \p NumElements is a valid matrix dimension.
+ static constexpr bool isDimensionValid(size_t NumElements) {
+ return NumElements > 0 && NumElements <= MaxElementsPerDimension;
+ }
+
+ /// Return true if \p Scope is valid
+ static constexpr bool isScopeValid(size_t Scope) {
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ }
+
+ /// Return true if \p Use is valid
+ static constexpr bool isUseValid(size_t Use) {
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ }
+
+ /// Returns the maximum number of elements per dimension.
+ static constexpr unsigned getMaxElementsPerDimension() {
+ return MaxElementsPerDimension;
+ }
+
+ void Profile(llvm::FoldingSetNodeID &ID) {
+ Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ getUse(), getTypeClass());
+ }
+
+ static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
+ unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Use, TypeClass TypeClass) {
+ ID.AddPointer(ElementType.getAsOpaquePtr());
+ ID.AddInteger(NumRows);
+ ID.AddInteger(NumColumns);
+ ID.AddInteger(Scope);
+ ID.AddInteger(Use);
+ ID.AddInteger(TypeClass);
+ }
+
+ static bool classof(const Type *T) {
+ return T->getTypeClass() == CooperativeMatrix;
+ }
+};
+
/// Represents a matrix type where the type and the number of rows and columns
/// is dependent on a template.
class DependentSizedMatrixType final : public MatrixType {
@@ -8851,6 +8937,10 @@ inline bool Type::isConstantMatrixType() const {
return isa<ConstantMatrixType>(CanonicalType);
}
+inline bool Type::isCooperativeMatrixType() const {
+ return isa<CooperativeMatrixType>(CanonicalType);
+}
+
inline bool Type::isOverflowBehaviorType() const {
return isa<OverflowBehaviorType>(CanonicalType);
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index b3cb4d0f336295..24fcda8f4c7e58 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,8 +2142,10 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
+ Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
+ Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2151,26 +2153,50 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^~~~~~~~~~~~~~~~~
+ /// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2187,8 +2213,10 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
}
};
@@ -2196,6 +2224,10 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
+class CooperativeMatrixTypeLoc
+ : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType> {};
+
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 1a2a3fda0dcff0..88d84d6d3234f5 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -263,6 +263,19 @@ let Class = ConstantMatrixType in {
}]>;
}
+let Class = CooperativeMatrixType in {
+ def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
+ def : Property<"numColumns", UInt32> {
+ let Read = [{ node->getNumColumns() }];
+ }
+ def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+
+ def : Creator<[{
+ return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ }]>;
+}
+
let Class = DependentSizedMatrixType in {
def : Property<"rows", ExprRef> {
let Read = [{ node->getRowExpr() }];
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 700a73f6696903..e6f396245070c6 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,6 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<MatrixType>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index 9f1da65e0f9401..c9ebac63390f5e 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -70,5 +70,6 @@ TYPE_BIT_CODE(HLSLInlineSpirv, HLSL_INLINE_SPIRV, 60)
TYPE_BIT_CODE(PredefinedSugar, PREDEFINED_SUGAR, 61)
TYPE_BIT_CODE(SubstBuiltinTemplatePack, SUBST_BUILTIN_TEMPLATE_PACK, 62)
TYPE_BIT_CODE(OverflowBehavior, OVERFLOWBEHAVIOR, 63)
+TYPE_BIT_CODE(CooperativeMatrix, COOPERATIVE_MATRIX, 64)
#undef TYPE_BIT_CODE
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 58a761f11925c0..6052ee26e96fb2 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2215,6 +2215,17 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
break;
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(T);
+ TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
+ // The internal layout of a matrix value is implementation defined.
+ // Initially be ABI compatible with arrays with respect to alignment and
+ // size.
+ Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
+ Align = ElementInfo.Align;
+ break;
+ }
+
case Type::Builtin:
switch (cast<BuiltinType>(T)->getKind()) {
default: llvm_unreachable("Unknown builtin type!");
@@ -3546,6 +3557,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
case Type::Pipe:
case Type::BitInt:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
OS << "?";
return;
@@ -4381,6 +4393,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
case Type::ExtVector:
case Type::DependentSizedExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::ObjCObject:
@@ -4928,6 +4941,45 @@ QualType ASTContext::getConstantMatrixType(
return QualType(New, 0);
}
+QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
+ unsigned Scope, unsigned NumRows,
+ unsigned NumColumns,
+ unsigned Use) const {
+ llvm::FoldingSetNodeID ID;
+ CooperativeMatrixType::Profile(ID, ElementTy, Scope, NumRows, NumColumns, Use,
+ Type::CooperativeMatrix);
+
+ assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
+ "need a valid element type");
+ assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
+ CooperativeMatrixType::isDimensionValid(NumColumns) &&
+ "need valid matrix dimensions");
+ assert(CooperativeMatrixType::isScopeValid(Scope) &&
+ "need valid matrix scope");
+ assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
+ void *InsertPos = nullptr;
+ if (CooperativeMatrixType *MTP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ return QualType(MTP, 0);
+
+ QualType Canonical;
+ if (!ElementTy.isCanonical()) {
+ Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
+ NumRows, NumColumns, Use);
+
+ CooperativeMatrixType *NewIP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ assert(!NewIP && "Matrix type shouldn't already exist in the map");
+ (void)NewIP;
+ }
+
+ auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
+ ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
+ CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ Types.push_back(New);
+ return QualType(New, 0);
+}
+
QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
Expr *RowExpr,
Expr *ColumnExpr,
@@ -9796,6 +9848,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
return;
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
if (NotEncodedT)
*NotEncodedT = T;
return;
@@ -10764,6 +10817,18 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
+/// areCompatMatrixTypes - Return true if the two specified matrix types are
+/// compatible.
+static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
+ assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
+ return LHS->getElementType() == RHS->getElementType() &&
+ LHS->getScope() == RHS->getScope() &&
+ LHS->getNumRows() == RHS->getNumRows() &&
+ LHS->getNumColumns() == RHS->getNumColumns() &&
+ LHS->getUse() == RHS->getUse();
+}
+
bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
QualType SecondVec) {
assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
@@ -12278,6 +12343,11 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
RHSCan->castAs<ConstantMatrixType>()))
return LHS;
return {};
+ case Type::CooperativeMatrix:
+ if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
+ return LHS;
+ return {};
case Type::ObjCObject: {
// Check if the types are assignment compatible.
// FIXME: This should be type compatibility, e.g. whether
@@ -14642,6 +14712,17 @@ static QualType getCommonNonSugarTypeNode(const ASTContext &Ctx, const Type *X,
return Ctx.getConstantMatrixType(getCommonElementType(Ctx, MX, MY),
MX->getNumRows(), MX->getNumColumns());
}
+ case Type::CooperativeMatrix: {
+ const auto *MX = cast<CooperativeMatrixType>(X),
+ *MY = cast<CooperativeMatrixType>(Y);
+ assert(MX->getScope() == MY->getScope());
+ assert(MX->getNumRows() == MY->getNumRows());
+ assert(MX->getNumColumns() == MY->getNumColumns());
+ assert(MX->getUse() == MY->getUse());
+ return Ctx.getCooperativeMatrixType(getCommonElementType(Ctx, MX, MY),
+ MX->getScope(), MX->getNumRows(),
+ MX->getNumColumns(), MX->getUse());
+ }
case Type::DependentSizedMatrix: {
const auto *MX = cast<DependentSizedMatrixType>(X),
*MY = cast<DependentSizedMatrixType>(Y);
@@ -14796,6 +14877,7 @@ static QualType getCommonSugarTypeNode(const ASTContext &Ctx, const Type *X,
CANONICAL_TYPE(ConstantArray)
CANONICAL_TYPE(ArrayParameter)
CANONICAL_TYPE(ConstantMatrix)
+ CANONICAL_TYPE(CooperativeMatrix)
CANONICAL_TYPE(Enum)
CANONICAL_TYPE(ExtVector)
CANONICAL_TYPE(FunctionNoProto)
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 4c5f2f60a90005..020564f3edfb79 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -510,6 +510,20 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns), Layout(Layout) {}
+CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
+ nColumns, use, canonType) {}
+
+CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : MatrixType(tc, matrixType, canonType), NumRows(nRows),
+ NumColumns(nColumns), Scope(scope), Use(use) {}
+
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -1286,6 +1300,18 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
T->getNumColumns(), T->getLayout());
}
+ QualType VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ QualType elementType = recurse(T->getElementType());
+ if (elementType.isNull())
+ return {};
+ if (elementType.getAsOpaquePtr() == T->getElementType().getAsOpaquePtr())
+ return QualType(T, 0);
+
+ return Ctx.getCooperativeMatrixType(elementType, T->getScope(),
+ T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ }
+
QualType VisitOverflowBehaviorType(const OverflowBehaviorType *T) {
QualType UnderlyingType = recurse(T->getUnderlyingType());
if (UnderlyingType.isNull())
@@ -2193,6 +2219,10 @@ class GetContainedDeducedTypeVisitor
return Visit(T->getElementType());
}
+ Type *VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+ }
+
Type *VisitFunctionProtoType(const FunctionProtoType *T) {
if (Syntactic && T->hasTrailingReturn())
return const_cast<FunctionProtoType *>(T);
@@ -3274,6 +3304,8 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
+ if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
+ return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
@@ -5166,6 +5198,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
return Cache::get(cast<VectorType>(T)->getElementType());
case Type::ConstantMatrix:
return Cache::get(cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return Cache::get(cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return Cache::get(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5268,6 +5302,9 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
case Type::ConstantMatrix:
return computeTypeLinkageInfo(
cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return computeTypeLinkageInfo(
+ cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return computeTypeLinkageInfo(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5472,6 +5509,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::FunctionProto:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 13929f6fe08005..34380e3c8965f8 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -280,6 +280,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -917,6 +918,20 @@ void TypePrinter::printConstantMatrixAfter(const ConstantMatrixType *T,
printAfter(T->getElementType(), OS);
}
+void TypePrinter::printCooperativeMatrixBefore(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printBefore(T->getElementType(), OS);
+ OS << " __attribute__((coop_mat(";
+ OS << T->getScope() << ", " << T->getNumRows() << ", ";
+ OS << T->getNumColumns() << ", " << T->getUse();
+ OS << ")))";
+}
+
+void TypePrinter::printCooperativeMatrixAfter(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printAfter(T->getElementType(), OS);
+}
+
void TypePrinter::printDependentSizedMatrixBefore(
const DependentSizedMatrixType *T, raw_ostream &OS) {
if (Policy.UseHLSLTypes) {
>From 62cba28b48b6b646a74bdb1a90bfb597093ab34c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:15:43 -0400
Subject: [PATCH 02/38] Patch 2 - Define extension macro
---
.../include/clang/Basic/OpenCLExtensions.def | 1 +
clang/lib/Headers/opencl-c-base.h | 23 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index f8af9b843f95f4..89e96dca85a59d 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,6 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
+OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 251894ac5a2630..7b3d239299c6e7 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -771,4 +771,27 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From aa1aacd18e2d3fddd6420897b49ca2a1d0958cc4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:16:39 -0400
Subject: [PATCH 03/38] Patch 3 - Add support for serialization, round-trip and
mangling
---
clang/lib/AST/ASTImporter.cpp | 11 +++++++++
clang/lib/AST/ASTStructuralEquivalence.cpp | 15 +++++++++++++
clang/lib/AST/ExprConstant.cpp | 1 +
clang/lib/AST/ItaniumMangle.cpp | 26 ++++++++++++++++++++++
clang/lib/AST/MicrosoftMangle.cpp | 21 +++++++++++++++++
clang/lib/Serialization/ASTReader.cpp | 10 +++++++++
clang/lib/Serialization/ASTWriter.cpp | 12 ++++++++++
7 files changed, 96 insertions(+)
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 4c296b6829b1b0..1bf29a062287bd 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -2116,6 +2116,17 @@ ExpectedType clang::ASTNodeImporter::VisitConstantMatrixType(
*ToElementTypeOrErr, T->getNumRows(), T->getNumColumns(), T->getLayout());
}
+ExpectedType clang::ASTNodeImporter::VisitCooperativeMatrixType(
+ const clang::CooperativeMatrixType *T) {
+ ExpectedType ToElementTypeOrErr = import(T->getElementType());
+ if (!ToElementTypeOrErr)
+ return ToElementTypeOrErr.takeError();
+
+ return Importer.getToContext().getCooperativeMatrixType(
+ *ToElementTypeOrErr, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+}
+
ExpectedType clang::ASTNodeImporter::VisitDependentAddressSpaceType(
const clang::DependentAddressSpaceType *T) {
Error Err = Error::success();
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 029e33ae0db760..8ec1100f9f4d89 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1151,6 +1151,21 @@ bool ASTStructuralEquivalence::isEquivalent(
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
+ const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
+ // The element types must be structurally equivalent and the number of rows
+ // and columns must match.
+ if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
+ Mat2->getElementType()) ||
+ Mat1->getScope() != Mat2->getScope() ||
+ Mat1->getNumRows() != Mat2->getNumRows() ||
+ Mat1->getNumColumns() != Mat2->getNumColumns() ||
+ Mat1->getUse() != Mat2->getUse())
+ return false;
+ break;
+ }
+
case Type::FunctionProto: {
const auto *Proto1 = cast<FunctionProtoType>(T1);
const auto *Proto2 = cast<FunctionProtoType>(T2);
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index d5b8d0ed889d12..dfd84f62ada258 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -16583,6 +16583,7 @@ GCCTypeClass EvaluateBuiltinClassifyType(QualType T,
case Type::BlockPointer:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::ObjCObject:
case Type::ObjCInterface:
case Type::ObjCObjectPointer:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 1a6f012199745c..8264a11c5f35ca 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2556,6 +2556,7 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -4532,6 +4533,31 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
Out << "E";
}
+void CXXNameMangler::mangleType(const CooperativeMatrixType *T) {
+ // Mangle cooperative matrix types as a vendor extended type:
+ // u<Len>coop_matI<Scope><Rows><Columns><Use><element type>E
+
+ mangleVendorType("coop_mat");
+
+ Out << "I";
+ auto &ASTCtx = getASTContext();
+ unsigned BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
+ llvm::APSInt Scope(BitWidth);
+ Scope = T->getScope();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Scope);
+ llvm::APSInt Rows(BitWidth);
+ Rows = T->getNumRows();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Rows);
+ llvm::APSInt Columns(BitWidth);
+ Columns = T->getNumColumns();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Columns);
+ llvm::APSInt Use(BitWidth);
+ Use = T->getUse();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Use);
+ mangleType(T->getElementType());
+ Out << "E";
+}
+
void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
// Mangle matrix types as a vendor extended type:
// u<Len>matrix_typeI<row expr><column expr><element type>E
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 74edee531e48a0..612979b002b596 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3810,6 +3810,27 @@ void MicrosoftCXXNameMangler::mangleType(const ConstantMatrixType *T,
mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
}
+void MicrosoftCXXNameMangler::mangleType(const CooperativeMatrixType *T,
+ Qualifiers quals, SourceRange Range) {
+ QualType EltTy = T->getElementType();
+
+ llvm::SmallString<64> TemplateMangling;
+ llvm::raw_svector_ostream Stream(TemplateMangling);
+ MicrosoftCXXNameMangler Extra(Context, Stream);
+
+ Stream << "?$";
+
+ Extra.mangleSourceName("__coop_mat");
+ Extra.mangleType(EltTy, Range, QMM_Escape);
+
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getScope()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumRows()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumColumns()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getUse()));
+
+ mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
+}
+
void MicrosoftCXXNameMangler::mangleType(const DependentSizedMatrixType *T,
Qualifiers quals, SourceRange Range) {
Error(Range.getBegin(), "dependent-sized matrix type") << Range;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 36b86afeeb32f9..ee39d6b883cffd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7672,6 +7672,16 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ TL.setAttrNameLoc(readSourceLocation());
+ TL.setAttrOperandParensRange(readSourceRange());
+ TL.setAttrScopeOperand(Reader.readExpr());
+ TL.setAttrRowOperand(Reader.readExpr());
+ TL.setAttrColumnOperand(Reader.readExpr());
+ TL.setAttrUseOperand(Reader.readExpr());
+}
+
void TypeLocReader::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 640047c2f9fb3d..835ee0ac49071a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -457,6 +457,18 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ addSourceLocation(TL.getAttrNameLoc());
+ SourceRange range = TL.getAttrOperandParensRange();
+ addSourceLocation(range.getBegin());
+ addSourceLocation(range.getEnd());
+ Record.AddStmt(TL.getAttrScopeOperand());
+ Record.AddStmt(TL.getAttrRowOperand());
+ Record.AddStmt(TL.getAttrColumnOperand());
+ Record.AddStmt(TL.getAttrUseOperand());
+}
+
void TypeLocWriter::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
>From df95f8c5aaed2836aae8166a7dd6261c79891918 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:21 -0400
Subject: [PATCH 04/38] Patch 4 - Add support for semantic analysis and
checking
---
clang/include/clang/Basic/Attr.td | 8 +
.../clang/Basic/DiagnosticSemaKinds.td | 54 +++
clang/include/clang/Sema/Sema.h | 34 +-
clang/lib/Sema/SemaChecking.cpp | 310 +++++++++++++++++-
clang/lib/Sema/SemaDecl.cpp | 32 ++
clang/lib/Sema/SemaExpr.cpp | 131 +++++++-
clang/lib/Sema/SemaLookup.cpp | 1 +
clang/lib/Sema/SemaTemplate.cpp | 5 +
clang/lib/Sema/SemaTemplateDeduction.cpp | 31 ++
clang/lib/Sema/SemaType.cpp | 78 ++++-
clang/lib/Sema/TreeTransform.h | 42 +++
11 files changed, 704 insertions(+), 22 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 4ef0bd2b5ca7c1..f66fc29fdac450 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3981,6 +3981,14 @@ def MatrixType : TypeAttr {
let PragmaAttributeSupport = 0;
}
+def CoopMatrixType : TypeAttr {
+ let Spellings = [Clang<"coop_mat">];
+ let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
+ let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
+ UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
+ let Documentation = [Undocumented];
+}
+
def Visibility : InheritableAttr {
let Clone = 0;
let Spellings = [GCC<"visibility">];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 66155d9780e644..300fda4bed280c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11988,6 +11988,60 @@ def ext_opencl_ext_vector_type_rgba_selector: ExtWarn<
def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
+// OpenCL C Cooperative Matrix.
+def err_invalid_coopmat_attr
+ : Error<"invalid argument of cooperative matrix attribute">;
+
+def err_coop_mat_not_primitive
+ : Error<"cooperative matrix attribute only supported for integer and "
+ "floating-point types">;
+
+def err_coop_mat_rol_col_size
+ : Error<"cooperative matrix row and column must have power-of-two size">;
+
+def err_coop_mem_layout_enum
+ : Error<"memory layout for cooperative matrix should be "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+
+def err_coop_element_and_pointer_type
+ : Error<"inconsistent between cooperative matrix element type and buffer "
+ "pointer type">;
+
+def err_coop_matrix_arg
+ : Error<"argument must be a valid cooperative matrix type">;
+
+def err_coop_matrix_assignment
+ : Error<"builtin return value should be assigned to cooperative matrix "
+ "type variable">;
+
+def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_A">;
+
+def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_B">;
+
+def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+
+def err_coop_matrix_element_type
+ : Error<"inconsistent cooperative matrix element type">;
+
+def err_coop_matrix_row_or_col_mismatch
+ : Error<"mismatch of cooperative matrix row or column">;
+
+def err_coop_matrix_use_type
+ : Error<"inconsistent cooperative matrix use type">;
+
+def err_unsupported_coopmat_binary_operator
+ : Error<"unsupported cooperative matrix binary operator">;
+
+def err_unsupported_coopmat_scalar_operator
+ : Error<"unsupported cooperative matrix scalar operator">;
+
+def err_ref_coopmat_var
+ : Error<"invalid reference of cooperative matrix variable">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 7fbcc4aae7d6c5..0847e05509cb41 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3112,6 +3112,34 @@ class Sema final : public SemaBase {
BuiltinCountedByRefKind K);
bool BuiltinCountedByRef(CallExpr *TheCall);
+ // Coop matrix handling.
+ void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc, CallExpr *call);
+ void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc);
+ bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
+ bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult);
+ void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc);
+ ExprResult BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatScalarOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ ExprResult BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+
// Matrix builtin handling.
ExprResult BuiltinMatrixTranspose(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
@@ -15370,8 +15398,10 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
+
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc);
+ SourceLocation AttrLoc, Expr *Scope = nullptr,
+ Expr *Use = nullptr, bool IsCoopMat = false);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
@@ -15394,6 +15424,8 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
+ bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+
/// Build a function type.
///
/// This routine checks the function type according to C++ rules and
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 0531dfa877fbdf..6a74740dec507a 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4154,6 +4154,27 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
break;
}
+ case Builtin::BIcoop_mat_load:
+ return BuiltinCoopMatrixLoad(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_store:
+ return BuiltinCoopMatrixStore(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_mulAdd:
+ return BuiltinCoopMatrixMulAdd(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ return BuiltinCoopMatrixBinaryOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_mul:
+ return BuiltinCoopMatrixScalarOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_neg:
+ return BuiltinCoopMatrixScalarUnaryOp(TheCall, TheCallResult);
+
case Builtin::BI__builtin_matrix_transpose:
return BuiltinMatrixTranspose(TheCall, TheCallResult);
@@ -17421,6 +17442,270 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
return false;
}
+// Check coop_mat_load/store buffer pointer.
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
+ unsigned PtrArgIdx) {
+ bool ArgError = false;
+ Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
+ ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
+ if (PtrConv.isInvalid())
+ return true;
+ PtrExpr = PtrConv.get();
+ TheCall->setArg(PtrArgIdx, PtrExpr);
+
+ auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
+ QualType ElementTy;
+ if (!PtrTy) {
+ ArgError = true;
+ } else {
+ ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
+ if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ ArgError = true;
+ }
+ }
+
+ if (ArgError) {
+ Diag(PtrExpr->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+ << PtrArgIdx + 1 << 0 << /* pointer to element ty */ 5 << /* no fp */ 0
+ << PtrExpr->getType();
+ }
+
+ return ArgError;
+}
+
+// Check coop_mat_load/store matrix element has same type with buffer pointer.
+void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
+ auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
+ if (!MTy) {
+ Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ return;
+ }
+
+ assert(isa<PointerType>(BufferType));
+ auto *PTy = BufferType->castAs<PointerType>();
+
+ if (MTy->getElementType().getUnqualifiedType() !=
+ PTy->getPointeeType().getUnqualifiedType())
+ Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+}
+
+void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
+
+ FunctionDecl *F = call->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (Fname->isStr("coop_mat_load"))
+ CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
+ MatrixLoc);
+}
+
+// Check coop_mat_load/store layout argument
+bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
+ bool ArgError = false;
+ DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LayoutExpr);
+ if (DR) {
+ const auto *ECDHS = dyn_cast<EnumConstantDecl>(DR->getDecl());
+ if (ECDHS) {
+ if (ECDHS->getInitVal() != 0 && ECDHS->getInitVal() != 1)
+ ArgError = true;
+ } else
+ ArgError = true;
+ } else
+ ArgError = true;
+
+ if (ArgError)
+ Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+
+ return ArgError;
+}
+
+ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
+ return ExprError();
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
+ return ExprError();
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 4))
+ return ExprError();
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ return ExprError();
+ CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
+ return ExprError();
+ return CallResult;
+}
+
+void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+ FunctionDecl *F = TheCall->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (!Fname->isStr("coop_mat_mulAdd"))
+ return;
+
+ auto MC = TheCall->getArg(2);
+ auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = MC->getType()->getAs<CooperativeMatrixType>();
+ auto Loc = TheCall->getBeginLoc();
+
+ if (!MOutTy)
+ Diag(Loc, diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!MOutTy || !M2Ty)
+ return;
+
+ if (MOutTy->getUse() != 2)
+ Diag(Loc, diag::err_coop_matrix_useACC);
+
+ if (MOutTy->getElementType().getUnqualifiedType() !=
+ M2Ty->getElementType().getUnqualifiedType())
+ Diag(Loc, diag::err_coop_matrix_element_type);
+
+ if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+}
+
+bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc) {
+ auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
+ auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
+ if (!M0Ty)
+ Diag(ALoc, diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(BLoc, diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty)
+ return true;
+
+ if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+
+ if (M0Ty->getUse() != M1Ty->getUse()) {
+ Diag(ALoc, diag::err_coop_matrix_use_type);
+ return true;
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType()) {
+ Diag(ALoc, diag::err_coop_matrix_element_type);
+ return true;
+ }
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+
+ CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
+ Arg1->getBeginLoc());
+
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
+ if (ATy->isIntegerType() && CTy->isIntegerType())
+ return true;
+ if (ATy->isFloatingType() && CTy->isFloatingType())
+ return true;
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ Expr *Arg2 = TheCall->getArg(2);
+
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto *M1Ty = Arg1->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = Arg2->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+ auto Loc1 = Arg1->getBeginLoc();
+
+ if (!M0Ty)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty || !M2Ty)
+ return ExprError();
+
+ if (M0Ty->getUse() != 0)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ if (M1Ty->getUse() != 1)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ if (M2Ty->getUse() != 2)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+
+ if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
+ M2Ty->getElementType()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+
+ if (M0Ty->getNumRows() != M2Ty->getNumRows())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
+ (M0Ty->getNumColumns() != M1Ty->getNumRows()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
@@ -17431,8 +17716,9 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
- if (!MType) {
+ auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
+ auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
+ if (!ConstMType && !CoopMType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17441,11 +17727,23 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- QualType ResultType = Context.getConstantMatrixType(
- MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
+ if (ConstMType) {
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
+ if (CoopMType) {
+ QualType ResultType = Context.getCooperativeMatrixType(
+ CoopMType->getElementType(),CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
+
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index fdbe1521165122..343d260fb1a26d 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14102,6 +14102,25 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
+// Return true if RHSExpr is a cooperative matrix builtin call.
+bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ if (!call)
+ return false;
+ FunctionDecl *F = call->getDirectCallee();
+ if (!F)
+ return false;
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ if (!Fname)
+ return false;
+ if (Fname->getName().starts_with("coop_mat") &&
+ !Fname->getName().starts_with("coop_mat_length"))
+ return true;
+
+ return false;
+}
+
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
llvm::scope_exit ResetDeclForInitializer([this]() {
if (!this->ExprEvalContexts.empty())
@@ -14620,6 +14639,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
+
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (!VDecl->getType()->isMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
}
void Sema::ActOnInitializerError(Decl *D) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 9b3384cdbdac8c..78bbb3768e7257 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4675,6 +4675,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::TemplateSpecialization:
@@ -8059,6 +8060,16 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
+ if (srcTy->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns();
+ }
+
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -11398,12 +11409,14 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
if (!IsDiv &&
- (LHSTy->isConstantMatrixType() || RHSTy->isConstantMatrixType()))
+ (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+ if (IsDiv && LHSTy->isCooperativeMatrixType() && RHSTy->isArithmeticType())
+ return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
@@ -11803,6 +11816,15 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -11949,6 +11971,15 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -14008,6 +14039,32 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ }
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -15738,6 +15795,47 @@ static bool needsConversionOfHalfVec(bool OpRequiresConversion, ASTContext &Ctx,
return HasVectorOfHalfType(E0) && (!E1 || HasVectorOfHalfType(E1));
}
+ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Add:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
+ Args);
+ case BO_Sub:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
+ Args);
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
+ Args);
+ case BO_Div:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_binary_operator));
+}
+
+ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_scalar_operator));
+}
+
ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
BinaryOperatorKind Opc, Expr *LHSExpr,
Expr *RHSExpr, bool ForFoldExpression) {
@@ -15774,6 +15872,17 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
if (getLangOpts().OpenCL) {
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
+ // Cooperative matrix support
+ if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ // Check matrix types for assignment.
+ if (BO_Assign == Opc) {
+ if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
+ RHSExpr->getBeginLoc()))
+ return ExprError();
+ } else
+ return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
@@ -15801,6 +15910,18 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (!LHSExpr->getType()->isMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
+ return ExprError();
+ }
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ assert(call);
+ call->setType(LHSExpr->getType());
+ CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ call);
+ CheckCoopMatrixMatMulOutput(call);
+ }
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
LHS.get()->getObjectKind() != OK_ObjCProperty) {
@@ -16460,6 +16581,10 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
+ if (Opc == UO_Minus && Ty->isMatrixType()) {
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
+ {InputExpr});
+ }
// The only legal unary operation for atomics is '&'.
if ((Opc != UO_AddrOf && Ty->isAtomicType()) ||
// OpenCL special types - image, sampler, pipe, and blocks are to be used
@@ -16538,7 +16663,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
Opc == UO_Plus && resultType->isPointerType())
break;
-
+ else if (getLangOpts().OpenCL && Opc == UO_Minus &&
+ resultType->isCooperativeMatrixType())
+ break;
return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
<< resultType << Input.get()->getSourceRange());
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index bc8a30d4e9dad4..f78c65d1deb0e1 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3265,6 +3265,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BitInt:
break;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 8e7433bc712813..09d27b04455545 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6393,6 +6393,11 @@ bool UnnamedLocalNoLinkageFinder::VisitConstantMatrixType(
return Visit(T->getElementType());
}
+bool UnnamedLocalNoLinkageFinder::VisitCooperativeMatrixType(
+ const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+}
+
bool UnnamedLocalNoLinkageFinder::VisitFunctionProtoType(
const FunctionProtoType* T) {
for (const auto &A : T->param_types()) {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 2257964a4310bb..f44bdf45d1c913 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2447,6 +2447,30 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
&DependentSizedMatrixType::getColumnExpr);
}
+ // (clang extension)
+ //
+ // T __attribute__((coop_mat_type(<integral constant>,
+ // <integral constant>,
+ // <integral constant>,
+ // <integral constant>)))
+ case Type::CooperativeMatrix: {
+ const auto *MP = P->castAs<CooperativeMatrixType>(),
+ *MA = A->getAs<CooperativeMatrixType>();
+ if (!MA)
+ return TemplateDeductionResult::NonDeducedMismatch;
+
+ // Check that the dimensions are the same
+ if (MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns()) {
+ return TemplateDeductionResult::NonDeducedMismatch;
+ }
+ // Perform deduction on element types.
+ return DeduceTemplateArgumentsByTypeMatch(
+ S, TemplateParams, MP->getElementType(), MA->getElementType(), Info,
+ Deduced, TDF, degradeCallPartialOrderingKind(POK),
+ /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
+ }
+
// (clang extension)
//
// T __attribute__(((address_space(N))))
@@ -7087,6 +7111,13 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *MatType = cast<CooperativeMatrixType>(T);
+ MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
+ Depth, Used);
+ break;
+ }
+
case Type::DependentSizedMatrix: {
const DependentSizedMatrixType *MatType = cast<DependentSizedMatrixType>(T);
MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 202bd486321f94..4dcf2c69025019 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2527,9 +2527,11 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc) {
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc, Expr *ScopeExpr,
+ Expr *UseExpr, bool IsCoopMat) {
+ if (!IsCoopMat)
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2610,6 +2612,26 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
+ if (IsCoopMat) {
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr)
+ << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
@@ -5973,6 +5995,23 @@ static void fillAtomicQualLoc(AtomicTypeLoc ATL, const DeclaratorChunk &Chunk) {
ATL.setParensRange(SourceRange());
}
+static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
+ const ParsedAttributesView &Attrs) {
+ for (const ParsedAttr &AL : Attrs) {
+ if (AL.getKind() == ParsedAttr::AT_CoopMatrixType) {
+ MTL.setAttrNameLoc(AL.getLoc());
+ MTL.setAttrScopeOperand(AL.getArgAsExpr(0));
+ MTL.setAttrRowOperand(AL.getArgAsExpr(1));
+ MTL.setAttrColumnOperand(AL.getArgAsExpr(2));
+ MTL.setAttrUseOperand(AL.getArgAsExpr(3));
+ MTL.setAttrOperandParensRange(SourceRange());
+ return;
+ }
+ }
+
+ llvm_unreachable("no matrix_type attribute found at the expected location!");
+}
+
namespace {
class TypeSpecLocFiller : public TypeLocVisitor<TypeSpecLocFiller> {
Sema &SemaRef;
@@ -6366,6 +6405,9 @@ namespace {
void VisitMatrixTypeLoc(MatrixTypeLoc TL) {
fillMatrixTypeLoc(TL, Chunk.getAttrs());
}
+ void VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
+ fillCooperativeMatrixTypeLoc(TL, Chunk.getAttrs());
+ }
void VisitTypeLoc(TypeLoc TL) {
llvm_unreachable("unsupported TypeLoc kind in declarator!");
@@ -8981,23 +9023,28 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S) {
- if (!S.getLangOpts().MatrixTypes) {
+ Sema &S, bool IsCoopMat = false) {
+ if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- if (Attr.getNumArgs() != 2) {
+ unsigned int NumAttrs = IsCoopMat ? 4 : 2;
+ if (Attr.getNumArgs() != NumAttrs) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << 2;
+ << Attr << NumAttrs;
return;
}
-
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ if (IsCoopMat) {
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
+ Scope, Use, /* IsCoopMat */ true);
+ if (!T.isNull())
+ CurType = T;
+ }
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9281,6 +9328,11 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
attr.setUsedAsTypeAttr();
break;
+ case ParsedAttr::AT_CoopMatrixType:
+ HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ attr.setUsedAsTypeAttr();
+ break;
+
case ParsedAttr::AT_WebAssemblyFuncref: {
if (!HandleWebAssemblyFuncrefAttr(state, type, attr))
attr.setUsedAsTypeAttr();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 73367d6bea063d..498551937b85dd 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1048,6 +1048,12 @@ class TreeTransform {
unsigned NumColumns,
SourceLocation AttributeLoc);
+ /// Build a new cooperative matrix type given the element type and
+ /// scope and use and dimensions.
+ QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use);
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
@@ -6312,6 +6318,34 @@ TreeTransform<Derived>::TransformConstantMatrixType(TypeLocBuilder &TLB,
return Result;
}
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformCooperativeMatrixType(
+ TypeLocBuilder &TLB, CooperativeMatrixTypeLoc TL) {
+ const CooperativeMatrixType *T = TL.getTypePtr();
+ QualType ElementType = getDerived().TransformType(T->getElementType());
+ if (ElementType.isNull())
+ return QualType();
+
+ QualType Result = TL.getType();
+ if (getDerived().AlwaysRebuild() || ElementType != T->getElementType()) {
+ Result = getDerived().RebuildCooperativeMatrixType(
+ ElementType, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ if (Result.isNull())
+ return QualType();
+ }
+
+ CooperativeMatrixTypeLoc NewTL = TLB.push<CooperativeMatrixTypeLoc>(Result);
+ NewTL.setAttrNameLoc(TL.getAttrNameLoc());
+ NewTL.setAttrOperandParensRange(TL.getAttrOperandParensRange());
+ NewTL.setAttrScopeOperand(TL.getAttrScopeOperand());
+ NewTL.setAttrRowOperand(TL.getAttrRowOperand());
+ NewTL.setAttrColumnOperand(TL.getAttrColumnOperand());
+ NewTL.setAttrUseOperand(TL.getAttrUseOperand());
+
+ return Result;
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::TransformDependentSizedMatrixType(
TypeLocBuilder &TLB, DependentSizedMatrixTypeLoc TL) {
@@ -18240,6 +18274,14 @@ QualType TreeTransform<Derived>::RebuildConstantMatrixType(
AttributeLoc);
}
+template <typename Derived>
+QualType TreeTransform<Derived>::RebuildCooperativeMatrixType(
+ QualType ElementType, unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use) {
+ return SemaRef.Context.getCooperativeMatrixType(ElementType, Scope, NumRows,
+ NumColumns, Use);
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::RebuildDependentSizedMatrixType(
QualType ElementType, Expr *RowExpr, Expr *ColumnExpr,
>From c490cfba7c0c02a4c29550143a41b355b56cc5d2 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:56 -0400
Subject: [PATCH 05/38] Patch 5 - Emit LLVM IR and intrinsics
---
clang/include/clang/Basic/Builtins.td | 67 +++++++
clang/lib/CodeGen/CGBuiltin.cpp | 277 ++++++++++++++++++++++++++
clang/lib/CodeGen/CGDebugInfo.cpp | 29 +++
clang/lib/CodeGen/CGDebugInfo.h | 1 +
clang/lib/CodeGen/CGExpr.cpp | 61 ++++++
clang/lib/CodeGen/CGExprScalar.cpp | 14 ++
clang/lib/CodeGen/CodeGenFunction.cpp | 2 +
clang/lib/CodeGen/CodeGenFunction.h | 9 +
clang/lib/CodeGen/CodeGenTBAA.cpp | 2 +-
clang/lib/CodeGen/CodeGenTypes.cpp | 33 ++-
clang/lib/CodeGen/ItaniumCXXABI.cpp | 2 +
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +
clang/lib/CodeGen/QualTypeMapper.h | 1 +
13 files changed, 502 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 7a59aaa133d522..ab73570c9bbcba 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,6 +1931,73 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
+// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+def CoopMatLoad : Builtin {
+ let Spellings = ["coop_mat_load"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatStore : Builtin {
+ let Spellings = ["coop_mat_store"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatMulAdd : Builtin {
+ let Spellings = ["coop_mat_mulAdd"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryAdd : Builtin {
+ let Spellings = ["coop_mat_binary_add"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinarySub : Builtin {
+ let Spellings = ["coop_mat_binary_sub"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryMul : Builtin {
+ let Spellings = ["coop_mat_binary_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryDiv : Builtin {
+ let Spellings = ["coop_mat_binary_div"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarMul : Builtin {
+ let Spellings = ["coop_mat_scalar_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarNeg : Builtin {
+ let Spellings = ["coop_mat_scalar_neg"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatInit : Builtin {
+ let Spellings = ["coop_mat_init"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatLength : Builtin {
+ let Spellings = ["coop_mat_length"];
+ let Attributes = [NoThrow, Const, CustomTypeChecking];
+ let Prototype = "unsigned int(...)";
+}
+
// "Overloaded" Atomic operator builtins. These are overloaded to support data
// types of i8, i16, i32, i64, and i128. The front-end sees calls to the
// non-suffixed version of these (which has a bogus type) and transforms them to
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 233609605fb216..6156212d20a663 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2759,6 +2759,54 @@ static void ClearPadding(CodeGenFunction &CGF, Address Src,
}
}
+static std::string getSPIRVBuiltinName(unsigned BuiltinID,
+ bool IsFloat = false) {
+ switch (BuiltinID) {
+ case Builtin::BIcoop_mat_load:
+ return "__spirv_CooperativeMatrixLoadKHR";
+ case Builtin::BIcoop_mat_store:
+ return "__spirv_CooperativeMatrixStoreKHR";
+ case Builtin::BIcoop_mat_mulAdd:
+ return "__spirv_CooperativeMatrixMulAddKHR";
+ case Builtin::BIcoop_mat_binary_add:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFAdd" : "__spirv_IAdd";
+ case Builtin::BIcoop_mat_binary_sub:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFSub" : "__spirv_ISub";
+ case Builtin::BIcoop_mat_binary_mul:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFMul" : "__spirv_IMul";
+ case Builtin::BIcoop_mat_binary_div:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFDiv" : "__spirv_IDiv";
+ case Builtin::BIcoop_mat_scalar_mul:
+ return "__spirv_CooperativeMatrixScalarMulKHR";
+ case Builtin::BIcoop_mat_scalar_neg:
+ return "__spirv_CooperativeMatrixScalarNeg";
+ case Builtin::BIcoop_mat_init:
+ return "__spirv_CompositeConstruct";
+ case Builtin::BIcoop_mat_length:
+ return "__spirv_CooperativeMatrixLengthKHR";
+ }
+ assert(0 && "Unexpected Builtin");
+ return "";
+}
+
+static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
+ CodeGenModule &CGM,
+ const CooperativeMatrixType *MTy) {
+ llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
+ MTy->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
+} // namespace
+
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
@@ -4620,6 +4668,235 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(Result);
}
+ case Builtin::BIcoop_mat_load: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address SrcAddr = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(SrcAddr, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_load without coop_mat output operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Layout = EmitScalarExpr(E->getArg(1));
+ auto Stride = EmitScalarExpr(E->getArg(2));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_store: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address Dst = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(Dst, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_store without coop_mat input operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Arg0 = EmitScalarExpr(E->getArg(1));
+ auto Layout = EmitScalarExpr(E->getArg(2));
+ auto Stride = EmitScalarExpr(E->getArg(3));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_mulAdd: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_mulAdd without coop_mat output operand");
+
+ auto Arg0 = E->getArg(0);
+ auto Arg1 = E->getArg(1);
+ auto Arg2 = E->getArg(2);
+ auto MA = EmitScalarExpr(Arg0);
+ auto MB = EmitScalarExpr(Arg1);
+ auto MC = EmitScalarExpr(Arg2);
+ const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
+ const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MATy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *CType = getTargetExtType(*this, CGM, MCTy);
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, CType, Int1Ty}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ llvm::Type *ElTy = ConvertType(MTy->getElementType());
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name = (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_mul: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = Arg1->getType();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_neg: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_init: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ auto Init = EmitScalarExpr(E->getArg(0));
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Init->getType()}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(InitFn, {Init});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_length: {
+ const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ ConvertType(E->getType()), {AType}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
case Builtin::BI__builtin_masked_load:
case Builtin::BI__builtin_masked_expand_load: {
llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 93428777b9b24c..408fd1ec25a1ef 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3937,6 +3937,33 @@ llvm::DIType *CGDebugInfo::CreateType(const ConstantMatrixType *Ty,
return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
}
+llvm::DIType *CGDebugInfo::CreateType(const CooperativeMatrixType *Ty,
+ llvm::DIFile *Unit) {
+ // FIXME: Create another debug type for matrices
+ // For the time being, it treats it like a nested ArrayType.
+
+ llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
+ uint64_t Size = CGM.getContext().getTypeSize(Ty);
+ uint32_t Align = getTypeAlignIfRequired(Ty, CGM.getContext());
+
+ // Create ranges for both dimensions.
+ llvm::SmallVector<llvm::Metadata *, 2> Subscripts;
+ auto *ColumnCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumColumns()));
+ auto *RowCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumRows()));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ ColumnCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ RowCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
+ return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
+}
+
llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
uint64_t Size;
uint32_t Align;
@@ -4396,6 +4423,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
return CreateType(cast<VectorType>(Ty), Unit);
case Type::ConstantMatrix:
return CreateType(cast<ConstantMatrixType>(Ty), Unit);
+ case Type::CooperativeMatrix:
+ return CreateType(cast<CooperativeMatrixType>(Ty), Unit);
case Type::ObjCObjectPointer:
return CreateType(cast<ObjCObjectPointerType>(Ty), Unit);
case Type::ObjCObject:
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 37520394812b44..f617623af3af40 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -244,6 +244,7 @@ class CGDebugInfo {
llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ConstantMatrixType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const CooperativeMatrixType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 8323a170a1dbbc..31251fdc3535c0 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7694,3 +7694,64 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
}
+
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
+ llvm::Value *ScalarVal,
+ QualType CoopMatQTy)
+{
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(
+ VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(
+ VecTy->getElementCount(),
+ ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
+ BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy)
+{
+ auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
+
+ switch (Opcode) {
+ case BO_Add:
+ return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
+ : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+
+ case BO_Sub:
+ return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
+ : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+
+ case BO_Mul:
+ // mat * mat → element-wise FMul/IMul
+ // mat * scalar → handled separately (see scalar path below)
+ if (!RHS->getType()->isVectorTy()) {
+ // mat * scalar: broadcast scalar then element-wise mul
+ llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
+ return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
+ : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ }
+ return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
+ : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+
+ case BO_Div:
+ if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+
+ default:
+ llvm_unreachable("Unsupported cooperative matrix binary op");
+ }
+}
\ No newline at end of file
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 7ecaa149f7b343..4802bdff0f8054 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -956,6 +956,12 @@ class ScalarExprEmitter
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
+ if (E->getType()->isCooperativeMatrixType()) { \
+ return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
+ CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), \
+ E->getType()); \
+ } \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
if (result && !promotionTy.isNull()) \
@@ -3684,6 +3690,14 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
+ QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ if (CompTy->isFloatingType())
+ return Builder.CreateFNeg(Val, "coopmat.fneg");
+ else
+ return Builder.CreateNeg(Val, "coopmat.ineg");
+ }
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 72a5808cb62298..9bdba60bd8563f 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -262,6 +262,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -2584,6 +2585,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 490267aaefd86c..7b69f7ff450108 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5191,6 +5191,15 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
+ /// Helper function for EmitCoopMatBinaryOp
+ llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index d33188db9da158..c0ba3582076415 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -332,7 +332,7 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert(isa<ConstantMatrixType>(Ty) &&
+ assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index daed8559cc62f5..6ac24c6554922e 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -125,7 +125,21 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
-
+ if (T->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent a Coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+ return CoopMatType;
+ }
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -704,6 +718,23 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
MT->getNumRows() * MT->getNumColumns());
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+
+ ResultType = CoopMatType;
+ break;
+ }
case Type::FunctionNoProto:
case Type::FunctionProto:
ResultType = ConvertFunctionTypeInternal(T);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 8e9aaa1313528a..75f1b595398dfd 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3894,6 +3894,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -4197,6 +4198,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 5044883a49ca2e..5850604f88207b 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -136,6 +136,12 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
MT->getNumRows() * MT->getNumColumns(),
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(QT);
+ return Builder.getArrayType(convertType(MT->getElementType()),
+ MT->getNumRows() * MT->getNumColumns(),
+ ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ }
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
case Type::BitInt: {
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index f6bb43482c705e..73d1753349fd40 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -48,6 +48,7 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
+ const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
>From fb104e6df911460b6bc2fa93d693cc8c6dee9efb Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:20:46 -0400
Subject: [PATCH 06/38] Add LIT tests and fix minor build issues
---
clang/include/clang/AST/TypeBase.h | 8 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 94 +++++++++++++++
.../test/Preprocessor/coop-mat-opencl-ext.cl | 63 ++++++++++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 66 ++++++++++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 60 +++++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 83 +++++++++++++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 114 ++++++++++++++++++
clang/tools/libclang/CIndex.cpp | 1 +
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ++++++
9 files changed, 524 insertions(+), 4 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen.cl
create mode 100644 clang/test/Preprocessor/coop-mat-opencl-ext.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-ast-utils.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema-neg.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-type-infra.cl
create mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 9b1c9ef646f1f5..b8146904612239 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4627,14 +4627,14 @@ class CooperativeMatrixType final : public MatrixType {
/// Return true if \p Scope is valid
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
}
/// Return true if \p Use is valid
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
}
/// Returns the maximum number of elements per dimension.
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
new file mode 100644
index 00000000000000..58b7a882fdc870
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -0,0 +1,94 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+//
+// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
+// SPIR-V intrinsic names.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
+// TargetExtType (visible in the call signature).
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// ---------------------------------------------------------------------------
+kernel void test_store(__global float *ptr, MatA_t a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
+
+// ---------------------------------------------------------------------------
+// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a; MatB_t b; MatC_t c; MatC_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+
+// ---------------------------------------------------------------------------
+// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// ---------------------------------------------------------------------------
+kernel void test_binary_add(__global float *ptr) {
+ MatA_t a; MatA_t b; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a + b;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_binary_add
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
+
+// ---------------------------------------------------------------------------
+// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// ---------------------------------------------------------------------------
+kernel void test_scalar_mul(__global float *ptr, float s) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a * s;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+
+// ---------------------------------------------------------------------------
+// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// ---------------------------------------------------------------------------
+kernel void test_unary_neg(__global float *ptr) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = -a;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_unary_neg
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarNeg
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
new file mode 100644
index 00000000000000..7181efbe7e2a88
--- /dev/null
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -0,0 +1,63 @@
+// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+//
+// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
+//
+// Tests: extension macro is predefined when enabled, extension can be
+// explicitly enabled/disabled via pragma, all four enum types and
+// their constants are visible under -finclude-default-header.
+
+// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E -dM %s \
+// RUN: | FileCheck %s --check-prefix=EXT
+
+// EXT: cl_ext_kernel_cooperative_matrix
+
+// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E %s \
+// RUN: | FileCheck %s --check-prefix=NOEXT
+
+// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+
+// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+void test_enum_constants(void) {
+ // coop_matrix_scope_t
+ coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
+ (void)s;
+
+ // coop_matrix_use_t
+ coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
+ coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
+ coop_matrix_use_t u2 = CLK_COOPERATIVE_MATRIX_ACCUMULATOR;
+ (void)u0; (void)u1; (void)u2;
+
+ // coop_matrix_layout_t
+ coop_matrix_layout_t l0 = CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR;
+ coop_matrix_layout_t l1 = CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR;
+ (void)l0; (void)l1;
+
+ // coop_matrix_operands_t
+ coop_matrix_operands_t op = CLK_COOPERATIVE_MATRIX_OPERAND_NONE;
+ (void)op;
+}
+
+// ── 2d. Enum constant values match the spec ──────────────────────────────────
+void test_enum_values(void) {
+ _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR == 0, "row major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR== 1, "col major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_OPERAND_NONE == 0, "operand none");
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
new file mode 100644
index 00000000000000..c6230eb94531f4
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -0,0 +1,66 @@
+// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
+//
+// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
+// ExprConstant, ItaniumMangle, MicrosoftMangle,
+// ASTReader/ASTWriter TypeLoc serialisation.
+//
+// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefix=MANGLE
+//
+// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch \
+// RUN: -ast-dump %t.pch_probe.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+//
+// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
+kernel void test_mangling(MatA_t a) { (void)a; }
+
+// MANGLE: @{{.*}}test_mangling{{.*}}(
+
+// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
+// must still carry a coop_mat parameter type.
+// PCH: FunctionDecl {{.*}} pch_probe
+// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
+
+// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_structural_equiv(void) {
+ MatA_t *p = 0;
+ MatA_alias *q = p; // same canonical type
+ (void)q;
+}
+
+// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
+// integer, float, pointer …). Using __builtin_classify_type on it
+// compiles without error; result is 0 (no_type_class).
+void test_expr_constant(MatA_t a) {
+ int cls = __builtin_classify_type(a);
+ (void)cls;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
new file mode 100644
index 00000000000000..f8080448b6df51
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -0,0 +1,60 @@
+// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+//
+// Patch 4: Sema — negative tests for diagnostic paths.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 2. Invalid use value (99 is not 0/1/2)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 3. Mismatched element types in coop_mat_mulAdd
+// a/b are float matrices, c is an int matrix — should fire element type
+// mismatch diagnostic.
+// Use the two-step declare-then-assign pattern so that the coop_mat_load
+// fixup path fires correctly and we only get the intended mulAdd error.
+// ---------------------------------------------------------------------------
+kernel void test_muladd_type_mismatch(__global float *fptr,
+ __global int *iptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+ a = coop_mat_load(fptr, ROW_MAJOR, 16);
+ b = coop_mat_load(fptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+
+ MatC_int_t result;
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
+// This intentionally uses single-step initialization because the error
+// fires precisely when the LHS is NOT a coop mat type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_assignment(__global float *ptr) {
+ float bad;
+ bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
+ (void)bad;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
new file mode 100644
index 00000000000000..cec1f278b0037a
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -0,0 +1,83 @@
+// clang/test/SemaOpenCL/coop-mat-sema.cl
+//
+// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
+// builtin validation, expr handling, TreeTransform.
+//
+// Valid code -- expected-no-diagnostics
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
+// match the LHS variable type. Declare first, then assign so that
+// Sema's IsCoopMatrixBuiltin path fires correctly.
+// ---------------------------------------------------------------------------
+kernel void test_load_store(__global float *ptr,
+ __global float *out_ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ coop_mat_store(out_ptr, a, ROW_MAJOR, 16);
+}
+
+// ---------------------------------------------------------------------------
+// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ MatC_t result;
+ result = coop_mat_mulAdd(a, b, c);
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4d. Binary element-wise operators (+, -)
+// ---------------------------------------------------------------------------
+void test_binary_ops(MatA_t a, MatA_t b) {
+ MatA_t r_add = a + b;
+ MatA_t r_sub = a - b;
+ (void)r_add; (void)r_sub;
+}
+
+// ---------------------------------------------------------------------------
+// 4e. Scalar multiply operator
+// ---------------------------------------------------------------------------
+void test_scalar_ops(MatA_t a, float s) {
+ MatA_t r = a * s;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4f. Unary minus
+// ---------------------------------------------------------------------------
+void test_unary_minus(MatA_t a) {
+ MatA_t r = -a;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
+// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// ---------------------------------------------------------------------------
+kernel void test_assignment(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
new file mode 100644
index 00000000000000..2e6700973e6113
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -0,0 +1,114 @@
+// clang/test/SemaOpenCL/coop_mat_type_infra.cl
+//
+// Patch 1: CooperativeMatrixType AST node and type infrastructure.
+//
+// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
+// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
+// RecursiveASTVisitor traversal, mergeTypes compatibility,
+// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -ast-dump %s \
+// RUN: | FileCheck %s --check-prefix=AST
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+
+// ---------------------------------------------------------------------------
+// Enum constants (from opencl-c-base.h via -finclude-default-header)
+// ---------------------------------------------------------------------------
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+// ---------------------------------------------------------------------------
+// 1. Basic type construction — four use roles, float element type.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_float16x16;
+typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
+
+// AST: TypedefDecl {{.*}} MatA_float16x16
+// AST: TypedefDecl {{.*}} MatB_float16x16
+// AST: TypedefDecl {{.*}} MatC_float16x16
+// AST: TypedefDecl {{.*}} MatA_int8x8
+
+// ---------------------------------------------------------------------------
+// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// ---------------------------------------------------------------------------
+void test_type_spelling(void) {
+ MatA_float16x16 a;
+ MatB_float16x16 b;
+ MatC_float16x16 c;
+ MatA_int8x8 d;
+}
+
+// AST: VarDecl {{.*}} a {{.*}}coop_mat(
+// AST: VarDecl {{.*}} b {{.*}}coop_mat(
+// AST: VarDecl {{.*}} c {{.*}}coop_mat(
+// AST: VarDecl {{.*}} d {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
+// float(4B)*16*16 = 1024 B = 8192 bits
+// int(4B)*8*8 = 256 B = 2048 bits
+// ---------------------------------------------------------------------------
+void test_sizeof(void) {
+ _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
+ _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
+}
+
+// ---------------------------------------------------------------------------
+// 4. Parameter / return type — type preserved through function boundary.
+// ---------------------------------------------------------------------------
+MatA_float16x16 test_param_return(MatA_float16x16 in) {
+ return in;
+}
+
+// AST: FunctionDecl {{.*}} test_param_return
+// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 5. TypeLoc operand traversal — all four operand slots populated.
+// ---------------------------------------------------------------------------
+void test_typeloc_operands(void) {
+ float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
+ (void)local_acc;
+}
+
+// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// ---------------------------------------------------------------------------
+void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
+ (void)x;
+}
+
+// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_merge_types(void) {
+ MatA_float16x16 *p = 0;
+ MatA_alias *q = p; // same canonical type — no diagnostic
+ (void)q;
+}
+
+// ---------------------------------------------------------------------------
+// 8. PCH serialisation round-trip.
+// ---------------------------------------------------------------------------
+// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 8d246d41f31f66..951fedea44ee4d 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1912,6 +1912,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
new file mode 100644
index 00000000000000..c4e206b76b7603
--- /dev/null
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -0,0 +1,39 @@
+; llvm/test/Verifier/coop_mat_verifier_fix.ll
+;
+; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
+;
+; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
+; ((A && B) <= 64). After the fix the parentheses are explicit.
+;
+; This test verifies that a valid @llvm.experimental.noalias.scope.decl
+; call with a 2-element scope list (no third argument) passes the verifier
+; without error, and that an invalid third argument (> 64 bits) is caught.
+;
+; ── 6a. Valid call — no third argument → verifier must accept ────────────────
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
+; RUN: llvm-as < %s | opt -passes=verify -disable-output
+;
+; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
+; RUN: llvm-as %s -o %t.bc
+; RUN: opt -passes=verify %t.bc -disable-output
+
+; VALID: @llvm.experimental.noalias.scope.decl
+
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+define void @test_no_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+; Third arg present and <= 64 bits wide — must also pass
+define void @test_valid_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain_a"}
>From 36e791e0c9644be16c595087042fbdec3c7e742c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 18:39:15 -0400
Subject: [PATCH 07/38] Fix formatting issues
---
clang/include/clang/Sema/Sema.h | 15 +++---
clang/lib/CodeGen/CGBuiltin.cpp | 76 +++++++++++++++------------
clang/lib/CodeGen/CGExpr.cpp | 63 ++++++++++------------
clang/lib/CodeGen/CGExprScalar.cpp | 12 ++---
clang/lib/Sema/SemaChecking.cpp | 39 +++++++-------
clang/lib/Sema/SemaExpr.cpp | 35 ++++++------
clang/lib/Sema/TreeTransform.h | 2 +-
clang/lib/Serialization/ASTReader.cpp | 3 +-
clang/lib/Serialization/ASTWriter.cpp | 3 +-
9 files changed, 120 insertions(+), 128 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 0847e05509cb41..bbaeb6ae372408 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3114,19 +3114,16 @@ class Sema final : public SemaBase {
// Coop matrix handling.
void CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc, CallExpr *call);
+ SourceLocation MatrixLoc, CallExpr *call);
void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc);
+ QualType BufferType,
+ SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
- ExprResult CallResult);
- ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
- ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 6156212d20a663..6f36cfa5b9b38b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2797,7 +2797,7 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
llvm::Type *Tys[] = {ElTy};
// Unsigned arguments for TargetExtType
unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
+ MTy->getUse()};
// Create a TargetExtType to represent the coop matrix type
llvm::TargetExtType *RetType = llvm::TargetExtType::get(
CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
@@ -4677,8 +4677,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_load without coop_mat output operand");
+ CGM.ErrorUnsupported(E, "coop_mat_load without coop_mat output operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Layout = EmitScalarExpr(E->getArg(1));
@@ -4692,7 +4691,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LoadFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
@@ -4709,8 +4709,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_store without coop_mat input operand");
+ CGM.ErrorUnsupported(E, "coop_mat_store without coop_mat input operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Arg0 = EmitScalarExpr(E->getArg(1));
@@ -4720,11 +4719,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
- VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
+ false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee StoreFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
@@ -4735,8 +4736,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_mulAdd: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_mulAdd without coop_mat output operand");
+ CGM.ErrorUnsupported(E,
+ "coop_mat_mulAdd without coop_mat output operand");
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
@@ -4760,11 +4761,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto *AType = getTargetExtType(*this, CGM, MATy);
auto *BType = getTargetExtType(*this, CGM, MBTy);
auto *CType = getTargetExtType(*this, CGM, MCTy);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee MatMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
@@ -4778,7 +4780,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_binary_div: {
auto Arg0 = EmitScalarExpr(E->getArg(0));
auto Arg1 = EmitScalarExpr(E->getArg(1));
-
+
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
@@ -4795,13 +4797,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
-
- std::string Name = (ElTy->isIntegerTy())
- ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
- : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
- llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name =
+ (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
@@ -4827,10 +4831,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
@@ -4854,10 +4859,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarNegFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
@@ -4870,10 +4876,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Init = EmitScalarExpr(E->getArg(0));
auto *RetType = getTargetExtType(*this, CGM, MTy);
// Set function type.
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {Init->getType()}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {Init->getType()}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee InitFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(InitFn, {Init});
@@ -4885,11 +4892,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
auto *AType = getTargetExtType(*this, CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- ConvertType(E->getType()), {AType}, false);
-
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
+
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LengthFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 31251fdc3535c0..45c1bf65645146 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7695,44 +7695,37 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
- llvm::Value *ScalarVal,
- QualType CoopMatQTy)
-{
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
-
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(
- VecTy->getElementCount(), C);
-
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(
- VecTy->getElementCount(),
- ScalarVal,
- "coopmat.broadcast");
-}
-
-llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
- BinaryOperatorKind Opcode,
- llvm::Value *LHS,
- llvm::Value *RHS,
- QualType ResultTy)
-{
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy) {
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy) {
auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
switch (Opcode) {
case BO_Add:
return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+ : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
case BO_Sub:
return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+ : Builder.CreateSub(LHS, RHS, "coopmat.isub");
case BO_Mul:
// mat * mat → element-wise FMul/IMul
@@ -7741,14 +7734,16 @@ llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
// mat * scalar: broadcast scalar then element-wise mul
llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
}
return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+ : Builder.CreateMul(LHS, RHS, "coopmat.imul");
case BO_Div:
- if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ if (IsFloat)
+ return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned)
+ return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
default:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 4802bdff0f8054..354143adf5c028 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -957,10 +957,9 @@ class ScalarExprEmitter
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
if (E->getType()->isCooperativeMatrixType()) { \
- return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
- CGF.EmitScalarExpr(E->getLHS()), \
- CGF.EmitScalarExpr(E->getRHS()), \
- E->getType()); \
+ return CGF.EmitCoopMatBinaryOp( \
+ E->getOpcode(), CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), E->getType()); \
} \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
@@ -3692,11 +3691,12 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ QualType CompTy =
+ E->getType()->getAs<CooperativeMatrixType>()->getElementType();
if (CompTy->isFloatingType())
return Builder.CreateFNeg(Val, "coopmat.fneg");
else
- return Builder.CreateNeg(Val, "coopmat.ineg");
+ return Builder.CreateNeg(Val, "coopmat.ineg");
}
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 6a74740dec507a..85b5e8aba592fb 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17443,8 +17443,7 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
}
// Check coop_mat_load/store buffer pointer.
-bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
- unsigned PtrArgIdx) {
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
bool ArgError = false;
Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
@@ -17475,8 +17474,8 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
// Check coop_mat_load/store matrix element has same type with buffer pointer.
void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc) {
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
@@ -17492,8 +17491,8 @@ void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
}
void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc,
- CallExpr *call) {
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
FunctionDecl *F = call->getDirectCallee();
assert(F);
@@ -17502,7 +17501,7 @@ void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
assert(Fname);
if (Fname->isStr("coop_mat_load"))
CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ MatrixLoc);
}
// Check coop_mat_load/store layout argument
@@ -17526,7 +17525,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
}
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
@@ -17545,7 +17544,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
return CallResult;
@@ -17584,7 +17583,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
- SourceLocation BLoc) {
+ SourceLocation BLoc) {
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
@@ -17613,7 +17612,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
}
ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17621,7 +17620,7 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ Arg1->getBeginLoc());
TheCall->setType(Arg0->getType());
@@ -17637,7 +17636,7 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
}
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
@@ -17685,7 +17684,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17696,7 +17695,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
return ExprError();
@@ -17729,17 +17728,17 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// matrix type.
if (ConstMType) {
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
}
if (CoopMType) {
QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(),CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
+ CoopMType->getElementType(), CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 78bbb3768e7257..33e9f21101b77d 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -11408,8 +11408,7 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
if (LHSTy->isSveVLSBuiltinType() || RHSTy->isSveVLSBuiltinType())
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
- if (!IsDiv &&
- (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
+ if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
@@ -14040,10 +14039,11 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return QualType();
if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
+ RHS.get()->getType()->isCooperativeMatrixType()) {
auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a matrix");
if (LHSMatType && RHSMatType) {
if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
return InvalidOperands(Loc, LHS, RHS);
@@ -14054,14 +14054,14 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
RHS.get()->getType().getUnqualifiedType());
QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
+ RHSELTy = RHSMatType->getElementType();
if (!Context.hasSameType(LHSELTy, RHSELTy))
return InvalidOperands(Loc, LHS, RHS);
return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
- LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
- LHSMatType->getUse());
+ Context.getCommonSugaredType(LHSELTy, RHSELTy),
+ LHSMatType->getScope(), LHSMatType->getNumRows(),
+ RHSMatType->getNumColumns(), LHSMatType->getUse());
}
}
@@ -15803,17 +15803,13 @@ ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Add:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add, Args);
case BO_Sub:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub, Args);
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul, Args);
case BO_Div:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div, Args);
default:
break;
}
@@ -15828,8 +15824,7 @@ ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul, Args);
default:
break;
}
@@ -15877,7 +15872,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
- RHSExpr->getBeginLoc()))
+ RHSExpr->getBeginLoc()))
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
@@ -15919,7 +15914,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
assert(call);
call->setType(LHSExpr->getType());
CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
+ call);
CheckCoopMatrixMatMulOutput(call);
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 498551937b85dd..eb7af59cc0faca 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1053,7 +1053,7 @@ class TreeTransform {
QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
unsigned NumRows, unsigned NumColumns,
unsigned Use);
-
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index ee39d6b883cffd..759176f58e46e3 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7672,8 +7672,7 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
-void TypeLocReader::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
TL.setAttrOperandParensRange(readSourceRange());
TL.setAttrScopeOperand(Reader.readExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 835ee0ac49071a..dcbabd39ad8f0e 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -457,8 +457,7 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
-void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
SourceRange range = TL.getAttrOperandParensRange();
addSourceLocation(range.getBegin());
>From 656fc37e6ca9b81e648def6a152b5c9fd7803959 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 15 Jul 2026 12:12:32 -0400
Subject: [PATCH 08/38] Add tests for coop_mat_init and coop_mat_length
---
.../coop-mat-init-length-codegen.cl | 112 ++++++++++++++++++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 94 +++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
new file mode 100644
index 00000000000000..b18f5617338c8f
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -0,0 +1,112 @@
+// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+// ---------------------------------------------------------------------------
+// Type definitions.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init — float scalar.
+// Expected IR: call __spirv_CompositeConstruct with the scalar value,
+// returning a spirv.CooperativeMatrixKHR TargetExtType.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_float
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float
+
+kernel void test_init_float(global float *out) {
+ MatA_t a;
+ a = coop_mat_init(1.0f);
+ (void)a;
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init — integer scalar.
+// Expected IR: call __spirv_CompositeConstruct with an i32 value.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_int
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(i32
+
+kernel void test_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+ (void)m;
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init — zero initialisation.
+// Expected IR: call __spirv_CompositeConstruct with 0.0.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_zero
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+
+kernel void test_init_zero(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ (void)acc;
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length — returns unsigned int.
+// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
+// an i32 alloca.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+
+kernel void test_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ *out = len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_init then coop_mat_length — combined flow.
+// Verifies the TargetExtType produced by init flows into length unchanged.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_then_length
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_init_then_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(2.0f);
+ *out = coop_mat_length(a);
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// The call must NOT be a plain 'call' — it must be 'call spir_func'.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length_cc
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_length_cc(global unsigned int *out) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ *out = coop_mat_length(acc);
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
new file mode 100644
index 00000000000000..ab499bc1b4af7e
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -0,0 +1,94 @@
+// Tests for coop_mat_init and coop_mat_length — Sema (positive).
+//
+// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
+// so arg-count and arg-type validation are not performed by Sema for these two
+// builtins. Only the positive (valid-code) path and the one assignment-target
+// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
+// tested here.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics (sections 1-6 must compile cleanly)
+
+// ---------------------------------------------------------------------------
+// Type definitions shared across all tests.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_B))) MatB_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init: initialise each use role with a float scalar.
+// Two-step (declare then assign) is required because the return type is
+// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_basic(void) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_init(1.0f);
+ b = coop_mat_init(2.0f);
+ c = coop_mat_init(0.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init: integer element type.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_chained(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ acc = coop_mat_init(1.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length: basic usage — returns unsigned int directly.
+// No two-step required for the length call itself.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_basic(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ (void)len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_length: all three use roles.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_roles(void) {
+ MatA_t a; a = coop_mat_init(0.0f);
+ MatB_t b; b = coop_mat_init(0.0f);
+ MatC_t c; c = coop_mat_init(0.0f);
+ unsigned int la = coop_mat_length(a);
+ unsigned int lb = coop_mat_length(b);
+ unsigned int lc = coop_mat_length(c);
+ (void)la; (void)lb; (void)lc;
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length: result used in arithmetic.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_arith(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int half_len = coop_mat_length(a) / 2u;
+ (void)half_len;
+}
>From 9158617a7e570e345998c897990e5189ed469b57 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 19:21:14 -0400
Subject: [PATCH 09/38] Add mangling support
Signed-off-by: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
---
clang/lib/CodeGen/CGBuiltin.cpp | 91 ++++++++-
.../coop-mat-codegen-mangling.cl | 174 ++++++++++++++++++
.../coop-mat-init-length-codegen.cl | 14 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 15 ++
4 files changed, 280 insertions(+), 14 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 6f36cfa5b9b38b..71fcca9f2fcd66 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2805,6 +2805,62 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
return RetType;
}
+/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
+/// TargetExtType: element type + scope + rows + cols + use.
+/// E.g. target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) ->
+/// "_f32_sc3_16x16_u2"
+static std::string getCoopMatFullSuffix(llvm::Type *Ty) {
+ auto *TET = cast<llvm::TargetExtType>(Ty);
+ llvm::Type *ElemTy = TET->getTypeParameter(0);
+
+ // Element type string
+ std::string ElemStr;
+ if (ElemTy->isFloatTy())
+ ElemStr = "f32";
+ else if (ElemTy->isHalfTy())
+ ElemStr = "f16";
+ else if (ElemTy->isDoubleTy())
+ ElemStr = "f64";
+ else if (auto *IntTy = dyn_cast<llvm::IntegerType>(ElemTy))
+ ElemStr = "i" + std::to_string(IntTy->getBitWidth());
+ else
+ llvm_unreachable("Unsupported cooperative matrix element type");
+
+ // Integer parameters: scope, rows, cols, use
+ unsigned Scope = TET->getIntParameter(0);
+ unsigned Rows = TET->getIntParameter(1);
+ unsigned Cols = TET->getIntParameter(2);
+ unsigned Use = TET->getIntParameter(3);
+
+ return "_" + ElemStr + "_sc" + std::to_string(Scope) + "_" +
+ std::to_string(Rows) + "x" + std::to_string(Cols) + "_u" +
+ std::to_string(Use);
+}
+
+/// Returns a name suffix encoding the address space of a pointer type,
+/// matching OpenCL/SPIR-V address space conventions.
+/// E.g. ptr addrspace(1) -> "_global"
+/// ptr addrspace(3) -> "_local"
+/// ptr addrspace(0) -> "_private"
+/// ptr addrspace(4) -> "_generic"
+static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
+ unsigned AS = cast<llvm::PointerType>(PtrTy)->getAddressSpace();
+ switch (AS) {
+ case 0:
+ return "_private";
+ case 1:
+ return "_global";
+ case 2:
+ return "_constant";
+ case 3:
+ return "_local";
+ case 4:
+ return "_generic";
+ default:
+ return "_as" + std::to_string(AS);
+ }
+}
+
} // namespace
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
@@ -4689,7 +4745,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy = llvm::FunctionType::get(
RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee LoadFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4722,7 +4781,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(1)->getType()));
llvm::FunctionCallee StoreFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4764,7 +4826,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(Arg0->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg1->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg2->getType()));
llvm::FunctionCallee MatMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
@@ -4804,6 +4870,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
(ElTy->isIntegerTy())
? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
: getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ // Function name mangling.
+ Name = Name + getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee BinaryFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
@@ -4833,7 +4901,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
@@ -4861,7 +4931,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarNegFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
@@ -4878,7 +4950,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee InitFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
@@ -4895,7 +4969,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(0)->getType()));
llvm::FunctionCallee LengthFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
new file mode 100644
index 00000000000000..9fec09287f890a
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -0,0 +1,174 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+//
+// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
+// Covers three mangling axes:
+// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
+// (B) coop_mat_mulAdd -- different element types (float vs int)
+// (C) coop_mat_load -- different pointer address spaces
+// (D) coop_mat_store -- different pointer address spaces
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
+// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
+// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+// ---------------------------------------------------------------------------
+// Matrix type aliases used across tests
+// ---------------------------------------------------------------------------
+
+// --- float 16x16 (MulAdd group 1) ---
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatF_16x16_A;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatF_16x16_B;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatF_16x16_C;
+
+// --- float 8x32 / 32x8 / 8x8 (MulAdd group 2 -- same elem type, diff dims) ---
+typedef float __attribute__((coop_mat(SCOPE, 8, 32, USE_A))) MatF_8x32_A;
+typedef float __attribute__((coop_mat(SCOPE, 32, 8, USE_B))) MatF_32x8_B;
+typedef float __attribute__((coop_mat(SCOPE, 8, 8, USE_C))) MatF_8x8_C;
+
+// --- int 16x16 (MulAdd group 3 -- different element type to group 1) ---
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatI_16x16_A;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatI_16x16_B;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatI_16x16_C;
+
+// ===========================================================================
+// (A) MulAdd: same element type (float), different matrix dimensions
+// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
+// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
+// ===========================================================================
+
+kernel void test_muladd_f32_16x16(__global float *ptr) {
+ MatF_16x16_A a;
+ MatF_16x16_B b;
+ MatF_16x16_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+kernel void test_muladd_f32_8x32(__global float *ptr) {
+ MatF_8x32_A a;
+ MatF_32x8_B b;
+ MatF_8x8_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 8);
+ b = coop_mat_load(ptr, ROW_MAJOR, 32);
+ c = coop_mat_load(ptr, ROW_MAJOR, 8);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_8x32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x32_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_32x8_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x8_u{{[0-9]+}}
+
+// ===========================================================================
+// (B) MulAdd: different element types -- float 16x16 vs int 16x16
+// Same dimensions, different elem type -> must produce two distinct symbols.
+// ===========================================================================
+
+kernel void test_muladd_i32_16x16(__global int *iptr) {
+ MatI_16x16_A a;
+ MatI_16x16_B b;
+ MatI_16x16_C c, r;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16);
+ b = coop_mat_load(iptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_i32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _i32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+// ===========================================================================
+// (C) Load: different pointer address spaces
+// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
+// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
+// ===========================================================================
+
+kernel void test_load_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_load_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (D) Store: different pointer address spaces
+// Same matrix type, __global vs __local -> two distinct symbols.
+// ===========================================================================
+
+kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (E) Negative: confirm NO cross-contamination between address space variants
+// i.e. the _global load function is NOT called for the _local case.
+// ===========================================================================
+
+kernel void test_no_cross_contamination(__global float *gptr,
+ __local float *lptr) {
+ MatF_16x16_A ag, al;
+ ag = coop_mat_load(gptr, ROW_MAJOR, 16);
+ al = coop_mat_load(lptr, ROW_MAJOR, 16);
+ (void)ag; (void)al;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_no_cross_contamination
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_global_f32_{{.*}}addrspace(3)
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
+
+// Verify distinct declare lines exist (these appear at end of IR).
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index b18f5617338c8f..8dd1aab2b2a139 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -26,7 +26,7 @@ typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
// CHECK-LABEL: @{{.*}}test_init_float
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_float(global float *out) {
MatA_t a;
@@ -41,7 +41,7 @@ kernel void test_init_float(global float *out) {
// CHECK-LABEL: @{{.*}}test_init_int
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(i32
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_int(void) {
MatA_int_t m;
@@ -56,7 +56,7 @@ kernel void test_init_int(void) {
// CHECK-LABEL: @{{.*}}test_init_zero
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+// CHECK-SAME: @__spirv_CompositeConstruct_{{.*}}(float {{.*}}0
kernel void test_init_zero(void) {
MatC_t acc;
@@ -71,7 +71,7 @@ kernel void test_init_zero(void) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
kernel void test_length(global unsigned int *out) {
@@ -88,8 +88,8 @@ kernel void test_length(global unsigned int *out) {
// CHECK-LABEL: @{{.*}}test_init_then_length
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: @__spirv_CompositeConstruct
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_init_then_length(global unsigned int *out) {
MatA_t a;
@@ -103,7 +103,7 @@ kernel void test_init_then_length(global unsigned int *out) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length_cc
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_length_cc(global unsigned int *out) {
MatC_t acc;
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 800f3a4d0c4df6..383f72d4e7c169 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -569,6 +569,21 @@ static bool isNonMangledOCLBuiltin(StringRef Name) {
}
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
+ // Cooperative-matrix builtins emitted by Clang CodeGen carry mangling
+ // suffixes to avoid IR function-signature collisions, e.g.:
+ // "__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global"
+ // Strip everything from the first '_' after "__spirv_CooperativeMatrix"
+ // so the name matches the canonical DemangledNativeBuiltin entry in
+ // SPIRVBuiltins.td (e.g. "__spirv_CooperativeMatrixLoadKHR").
+ static constexpr StringRef CoopMatPrefix = "__spirv_CooperativeMatrix";
+ if (Name.starts_with(CoopMatPrefix)) {
+ size_t SuffixPos = Name.find('_', CoopMatPrefix.size());
+ if (SuffixPos != StringRef::npos)
+ return Name.substr(0, SuffixPos).str();
+ // No mangling suffix present — return the name as-is.
+ return Name.str();
+ }
+
bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
bool IsNonMangledSPIRV = Name.starts_with("__spirv_");
bool IsNonMangledHLSL = Name.starts_with("__hlsl_");
>From 9f6ea0fe2a7b5504ea3bdce1d3006d7a4e7b36e5 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 13:23:45 -0400
Subject: [PATCH 10/38] Change name of extension to cl_khr_cooperative_matrix
---
clang/include/clang/Basic/Builtins.td | 2 +-
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
.../test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../CodeGenOpenCL/coop-mat-init-length-codegen.cl | 2 +-
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 12 ++++++------
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 8 ++++----
clang/test/SemaOpenCL/coop-mat-init-length-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 6 +++---
11 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index ab73570c9bbcba..66c214440a99f7 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,7 +1931,7 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
-// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+// Cooperative Matrix builtins for cl_khr_cooperative_matrix
def CoopMatLoad : Builtin {
let Spellings = ["coop_mat_load"];
let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 89e96dca85a59d..8a583ad504c706 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9fec09287f890a..9d55bd8c61cc3c 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -8,7 +8,7 @@
// (D) coop_mat_store -- different pointer address spaces
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 58b7a882fdc870..ad8df2939269f5 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -4,7 +4,7 @@
// SPIR-V intrinsic names.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 8dd1aab2b2a139..dfa763b060e720 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -1,7 +1,7 @@
// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 7181efbe7e2a88..f77fe542fd210f 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,6 +1,6 @@
// clang/test/Preprocessor/coop_mat_opencl_ext.cl
//
-// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// Patch 2: cl_khr_cooperative_matrix registration in
// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
//
// Tests: extension macro is predefined when enabled, extension can be
@@ -9,23 +9,23 @@
// ── 2a. Extension macro is predefined when the extension is enabled ─────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
// RUN: | FileCheck %s --check-prefix=EXT
-// EXT: cl_ext_kernel_cooperative_matrix
+// EXT: cl_khr_cooperative_matrix
// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
// RUN: | FileCheck %s --check-prefix=NOEXT
-// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+// NOEXT-NOT: cl_khr_cooperative_matrix
// ── 2c. Enum constants are visible when extension is enabled ─────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index c6230eb94531f4..e8fad815806ece 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -6,24 +6,24 @@
//
// ── 3a. Itanium mangling ────────────────────────────────────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch \
// RUN: -ast-dump %t.pch_probe.cl \
// RUN: | FileCheck %s --check-prefix=PCH
//
// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index ab499bc1b4af7e..799f7a9e15a352 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -7,7 +7,7 @@
// tested here.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics (sections 1-6 must compile cleanly)
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index f8080448b6df51..0c27978d492a92 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -3,7 +3,7 @@
// Patch 4: Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index cec1f278b0037a..bcedd94fdbf17d 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -5,7 +5,7 @@
//
// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 2e6700973e6113..43adeebd945006 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -8,16 +8,16 @@
// TypeLoc operand slots, sizeof / getTypeInfoImpl.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -ast-dump %s \
// RUN: | FileCheck %s --check-prefix=AST
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
>From bfc50469090fbaaf8bffcd24dde434245d030095 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 14:37:44 -0400
Subject: [PATCH 11/38] Fix test fails
---
clang/lib/Sema/SemaType.cpp | 6 ++++++
clang/test/AST/undocumented-attrs.cpp | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 4dcf2c69025019..476f4d24fc1ecf 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9044,6 +9044,12 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Scope, Use, /* IsCoopMat */ true);
if (!T.isNull())
CurType = T;
+ } else {
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
}
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index 525466b1efdc0c..dff9d3845062a2 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,6 +27,7 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
+CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -88,4 +89,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 82
+CHECK-NEXT: Total: 83
>From e377f66b8f8e0fd2038acd085d24aa4bcd0ca650 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sun, 9 Aug 2026 00:57:18 -0400
Subject: [PATCH 12/38] Fix failing lldb build and a couple of tests
---
.../test/Misc/pragma-attribute-supported-attributes-list.test | 1 +
lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 3 +++
llvm/test/Verifier/coop-mat-verifier-fix.ll | 4 ++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 8f8c1aa3d3f9cd..c906921cc1b7a4 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -63,6 +63,7 @@
// CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record)
// CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record)
// CHECK-NEXT: Convergent (SubjectMatchRule_function)
+// CHECK-NEXT: CoopMatrixType (SubjectMatchRule_type_alias, SubjectMatchRule_variable, SubjectMatchRule_variable_is_parameter, SubjectMatchRule_field)
// CHECK-NEXT: CoroAwaitElidable (SubjectMatchRule_record)
// CHECK-NEXT: CoroAwaitElidableArgument (SubjectMatchRule_variable_is_parameter)
// CHECK-NEXT: CoroDisableLifetimeBound (SubjectMatchRule_function)
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index f3e8baaf38453d..9d2f337e856acd 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -4224,6 +4224,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
// Matrix types that we're not sure how to display at the moment.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5101,6 +5102,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type) {
break;
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5274,6 +5276,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
// Matrix types we're not sure how to display yet.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
index c4e206b76b7603..7f2eedb657d41d 100644
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -22,14 +22,14 @@
declare void @llvm.experimental.noalias.scope.decl(metadata)
define void @test_no_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
; Third arg present and <= 64 bits wide — must also pass
define void @test_valid_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
>From 03fe6d2442b05e12ec35c5dd6a7934e2489220b6 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 4 Sep 2026 18:58:24 -0400
Subject: [PATCH 13/38] Minor changes to sync with latest upstream changes
---
clang/lib/AST/ASTContext.cpp | 10 ++++------
clang/lib/CodeGen/CGBuiltin.cpp | 2 --
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 6052ee26e96fb2..a7e096a4339cfb 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4957,9 +4957,8 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
- void *InsertPos = nullptr;
- if (CooperativeMatrixType *MTP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ llvm::FoldingSetInsertToken Token;
+ if (CooperativeMatrixType *MTP = CooperativeMatrixTypes.lookup(ID, Token))
return QualType(MTP, 0);
QualType Canonical;
@@ -4967,15 +4966,14 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
NumRows, NumColumns, Use);
- CooperativeMatrixType *NewIP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ CooperativeMatrixType *NewIP = CooperativeMatrixTypes.lookup(ID, Token);
assert(!NewIP && "Matrix type shouldn't already exist in the map");
(void)NewIP;
}
auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
- CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ CooperativeMatrixTypes.insert(New, Token);
Types.push_back(New);
return QualType(New, 0);
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 71fcca9f2fcd66..b6d79bd240f065 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2861,8 +2861,6 @@ static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
}
}
-} // namespace
-
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
>From e20d4d8ac9e037ab38b460572d65e1dbb1a2a29c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 13:55:14 -0400
Subject: [PATCH 14/38] Update CooperativeMatrixType class definition to be
derived directly from Type class, instead of MatrixType class.
---
clang/include/clang/AST/OperationKinds.def | 3 +
clang/include/clang/AST/TypeBase.h | 78 ++++++---
clang/include/clang/AST/TypeLoc.h | 105 +++++++----
clang/include/clang/AST/TypeProperties.td | 3 +
clang/include/clang/Basic/TypeNodes.td | 2 +-
clang/include/clang/CIR/Dialect/IR/CIROps.td | 1 +
clang/include/clang/Sema/Sema.h | 27 ++-
clang/lib/AST/ASTContext.cpp | 18 +-
clang/lib/AST/Expr.cpp | 1 +
clang/lib/AST/ExprConstant.cpp | 2 +
clang/lib/AST/Type.cpp | 16 --
clang/lib/CIR/CodeGen/CIRGenExpr.cpp | 2 +
clang/lib/CodeGen/CGExpr.cpp | 78 +++++----
clang/lib/CodeGen/CGExprAgg.cpp | 2 +
clang/lib/CodeGen/CGExprComplex.cpp | 1 +
clang/lib/CodeGen/CGExprConstant.cpp | 1 +
clang/lib/CodeGen/CGExprScalar.cpp | 16 +-
clang/lib/CodeGen/CodeGenFunction.h | 23 ++-
clang/lib/CodeGen/CodeGenTBAA.cpp | 5 +-
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +-
clang/lib/CodeGen/QualTypeMapper.h | 1 -
clang/lib/Edit/RewriteObjCFoundationAPI.cpp | 1 +
clang/lib/Sema/SemaCast.cpp | 16 ++
clang/lib/Sema/SemaChecking.cpp | 32 ++--
clang/lib/Sema/SemaDecl.cpp | 2 +-
clang/lib/Sema/SemaExpr.cpp | 175 ++++++++++++++-----
clang/lib/Sema/SemaTemplateDeduction.cpp | 8 +-
clang/lib/Sema/SemaType.cpp | 145 ++++++++++-----
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 26 +--
clang/tools/libclang/CIndex.cpp | 2 +-
30 files changed, 521 insertions(+), 277 deletions(-)
diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def
index 8a13ad988403bf..d0cda0551f2d52 100644
--- a/clang/include/clang/AST/OperationKinds.def
+++ b/clang/include/clang/AST/OperationKinds.def
@@ -185,6 +185,9 @@ CAST_OPERATION(ToVoid)
/// CK_MatrixCast - A cast between matrix types of the same dimensions.
CAST_OPERATION(MatrixCast)
+/// CK_CoopMatrixCast - A cast between compatible cooperative matrix types.
+CAST_OPERATION(CoopMatrixCast)
+
/// CK_VectorSplat - A conversion from an arithmetic type to a
/// vector of that element type. Fills all elements ("splats") with
/// the source value.
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index b8146904612239..0b83ac3fd03e40 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4483,7 +4483,6 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
- T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4579,65 +4578,81 @@ class ConstantMatrixType final : public MatrixType {
}
};
-/// Represents a cooperative matrix type.
-class CooperativeMatrixType final : public MatrixType {
+/// Represents an opaque OpenCL cooperative matrix type.
+///
+/// Unlike MatrixType, a cooperative matrix is not an ordinary matrix value.
+/// It is an opaque, distributed object whose shape and role are part of its
+/// type identity.
+class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
protected:
friend class ASTContext;
+ /// Element type of the cooperative matrix.
+ QualType ElementType;
+
/// Number of rows and columns.
unsigned NumRows;
unsigned NumColumns;
- /// Scope and use
+ /// Cooperative matrix scope and use.
unsigned Scope;
unsigned Use;
static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
- CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
- unsigned NRows, unsigned NColumns, unsigned Use,
- QualType CanonElementType);
+ CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
+ unsigned NumColumns, unsigned Use,
+ QualType CanonicalType)
+ : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
+ NumRows, NumColumns, Use, CanonicalType) {}
- CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
- unsigned Scope, unsigned NRows, unsigned NColumns,
- unsigned Use, QualType CanonElementType);
+ CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use, QualType CanonicalType)
+ : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
+ Scope(Scope), Use(Use) {}
public:
- /// Returns the number of rows in the matrix.
+ /// Returns the element type.
+ QualType getElementType() const { return ElementType; }
+
+ /// Returns the number of rows.
unsigned getNumRows() const { return NumRows; }
- /// Returns the number of columns in the matrix.
+ /// Returns the number of columns.
unsigned getNumColumns() const { return NumColumns; }
- /// Returns the scope of the matrix.
+ /// Returns the cooperative matrix scope.
unsigned getScope() const { return Scope; }
- /// Returns the use of the matrix.
+ /// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into a vector.
+ /// Returns the number of elements required to embed the matrix into
+ /// a vector representation.
unsigned getNumElementsFlattened() const {
return getNumRows() * getNumColumns();
}
- /// Returns true if \p NumElements is a valid matrix dimension.
+ /// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
}
- /// Return true if \p Scope is valid
+ /// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
+ return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
}
- /// Return true if \p Use is valid
+ /// Returns true if \p Use is a valid cooperative matrix use.
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
+ return Use == 0 || // CLK_COOPERATIVE_MATRIX_A
+ Use == 1 || // CLK_COOPERATIVE_MATRIX_B
+ Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum number of elements per dimension.
+ /// Returns the maximum valid number of elements per dimension.
static constexpr unsigned getMaxElementsPerDimension() {
return MaxElementsPerDimension;
}
@@ -4658,6 +4673,23 @@ class CooperativeMatrixType final : public MatrixType {
ID.AddInteger(TypeClass);
}
+ static bool isValidElementType(QualType ElemTy) {
+ return ElemTy->isSpecificBuiltinType(BuiltinType::Char_S) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UChar) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Short) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UShort) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Int) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UInt) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Long) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::ULong) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Half) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Float) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Double);
+ }
+
+ bool isSugared() const { return false; }
+ QualType desugar() const { return QualType(this, 0); }
+
static bool classof(const Type *T) {
return T->getTypeClass() == CooperativeMatrix;
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 24fcda8f4c7e58..122090f7e6ec57 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,10 +2142,8 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
- Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
- Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2153,50 +2151,26 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
+ /// ^~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
- void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
-
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
- void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
-
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2213,10 +2187,8 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
- setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
- setAttrUseOperand(nullptr);
}
};
@@ -2224,15 +2196,82 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
-class CooperativeMatrixTypeLoc
- : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
- CooperativeMatrixType> {};
-
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
DependentSizedMatrixType> {};
+struct CooperativeMatrixTypeLocInfo {
+ SourceLocation AttrLoc;
+ SourceRange OperandParens;
+ Expr *ScopeOperand;
+ Expr *RowOperand;
+ Expr *ColumnOperand;
+ Expr *UseOperand;
+};
+
+class CooperativeMatrixTypeLoc
+ : public ConcreteTypeLoc<UnqualTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType,
+ CooperativeMatrixTypeLocInfo> {
+public:
+ /// The location of the attribute name, i.e.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^^~~~~~~~~~~~~~~~~~~
+ SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
+ void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+
+ /// The attribute's scope operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
+ /// The attribute's row operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
+ void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
+
+ /// The attribute's column operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
+ void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+
+ /// The attribute's use operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
+ /// The location of the parentheses around the operand, if there is
+ /// an operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^
+ SourceRange getAttrOperandParensRange() const {
+ return getLocalData()->OperandParens;
+ }
+ void setAttrOperandParensRange(SourceRange range) {
+ getLocalData()->OperandParens = range;
+ }
+
+ SourceRange getLocalSourceRange() const {
+ SourceRange range(getAttrNameLoc());
+ range.setEnd(getAttrOperandParensRange().getEnd());
+ return range;
+ }
+
+ void initializeLocal(ASTContext &Context, SourceLocation loc) {
+ setAttrNameLoc(loc);
+ setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
+ setAttrRowOperand(nullptr);
+ setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
+ }
+};
+
// FIXME: location of the '_Complex' keyword.
class ComplexTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
ComplexTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 88d84d6d3234f5..d134edc39e92ce 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -264,6 +264,9 @@ let Class = ConstantMatrixType in {
}
let Class = CooperativeMatrixType in {
+ def : Property<"elementType", QualType> {
+ let Read = [{ node->getElementType() }];
+ }
def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index e6f396245070c6..28e4b94c3bea92 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,7 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
-def CooperativeMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<Type>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 985cc3712c581d..ddbf6cf17ac8fb 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -213,6 +213,7 @@ def CIR_CastKind : CIR_I32Enum<"CastKind", "cast kind", [
I32EnumCase<"ptr_to_bool", 23>,
// CK_ToVoid
// CK_MatrixCast
+ // CK_CoopMatrixCast
// CK_VectorSplat
I32EnumCase<"integral", 27>,
I32EnumCase<"int_to_bool", 28>,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index bbaeb6ae372408..6853a2fd2e8698 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8025,6 +8025,14 @@ class Sema final : public SemaBase {
QualType CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
SourceLocation Loc, bool IsCompAssign);
+ /// Type checking for cooperative matrix binary operators.
+ QualType CheckCoopMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+ QualType CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+
/// Are the two types SVE-bitcast-compatible types? I.e. is bitcasting from
/// the first SVE type (e.g. an SVE VLAT) to the second type (e.g. an SVE
/// VLST) allowed?
@@ -8037,6 +8045,11 @@ class Sema final : public SemaBase {
/// do they have the same number of rows and the same number of columns?
bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ /// Are the two types cooperative matrix types and do they have the same
+ /// dimensions i.e. do they have the same number of rows and the same number
+ /// of columns? Also do they have the same scope and use?
+ bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+
bool areVectorTypesSameSize(QualType srcType, QualType destType);
/// Are the two types lax-compatible vector types? That is, given
@@ -8079,6 +8092,13 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
+ // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
+ // We allow casting between cooperative matrixes of the same scope, use, and
+ // same dimensions i.e. when they have the same number of rows and columns.
+ // Returns true if the cast is invalid.
+ bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind);
+
// CheckVectorCast - check type constraints for vectors.
// Since vectors are an extension, there are no C standard reference for this.
// We allow casting between vectors and integer datatypes of the same size.
@@ -15397,8 +15417,11 @@ class Sema final : public SemaBase {
SourceLocation AttrLoc);
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc, Expr *Scope = nullptr,
- Expr *Use = nullptr, bool IsCoopMat = false);
+ SourceLocation AttrLoc);
+
+ QualType BuildCoopMatrixType(QualType T, Expr *Scope, Expr *Use,
+ Expr *NumRows, Expr *NumColumns,
+ SourceLocation AttrLoc);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index a7e096a4339cfb..0317ce1268150a 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2218,9 +2218,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
case Type::CooperativeMatrix: {
const auto *MT = cast<CooperativeMatrixType>(T);
TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
- // The internal layout of a matrix value is implementation defined.
- // Initially be ABI compatible with arrays with respect to alignment and
- // size.
+ // The internal layout of a cooperative matrix value is implementation
+ // defined. Initially be ABI compatible with arrays with respect to
+ // alignment and size.
Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
Align = ElementInfo.Align;
break;
@@ -10815,10 +10815,10 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatMatrixTypes - Return true if the two specified matrix types are
-/// compatible.
-static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
- const CooperativeMatrixType *RHS) {
+/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
+/// matrix types are compatible.
+static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
return LHS->getElementType() == RHS->getElementType() &&
LHS->getScope() == RHS->getScope() &&
@@ -12342,8 +12342,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
return LHS;
return {};
case Type::CooperativeMatrix:
- if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
- RHSCan->castAs<CooperativeMatrixType>()))
+ if (areCompatCoopMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
return LHS;
return {};
case Type::ObjCObject: {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 6b9aef5d479a61..db07fa0be8387f 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1950,6 +1950,7 @@ bool CastExpr::CastConsistency() const {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
assert(!getType()->isBooleanType() && "unheralded conversion to bool");
goto CheckNoBasePath;
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index dfd84f62ada258..45125f020612a8 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -20212,6 +20212,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointCast:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLAggregateSplatCast:
llvm_unreachable("invalid cast kind for integral value");
@@ -21144,6 +21145,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLElementwiseCast:
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 020564f3edfb79..1c4a648003846e 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -510,20 +510,6 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns), Layout(Layout) {}
-CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
- nColumns, use, canonType) {}
-
-CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : MatrixType(tc, matrixType, canonType), NumRows(nRows),
- NumColumns(nColumns), Scope(scope), Use(use) {}
-
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -3304,8 +3290,6 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
- if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
- return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index 51533f9dc0f9c5..6869facf0d95c1 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -259,6 +259,7 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
case CK_IntegralToFloating:
case CK_LValueBitCast:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_MemberPointerToBoolean:
case CK_NonAtomicToAtomic:
case CK_ObjCObjectLValueCast:
@@ -1688,6 +1689,7 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 45c1bf65645146..ecd89bd18a0db8 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7695,58 +7695,68 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy) {
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+llvm::Value *
+CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy) {
+ assert(!Args.empty() &&
+ "Expected at least one argument for cooperative matrix builtin");
+
+ llvm::LLVMContext &Context = CGM.getLLVMContext();
+ llvm::Module &Module = CGM.getModule();
+
+ llvm::Type *ResultLLVMType = ConvertType(ResultTy);
+
+ SmallVector<llvm::Type *, 8> ArgTypes;
+ ArgTypes.reserve(Args.size());
+
+ for (llvm::Value *Arg : Args) {
+ assert(Arg && "Cooperative matrix builtin argument must not be null");
+ ArgTypes.push_back(Arg->getType());
+ }
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+ // Although the frontend builtin is declared as void(...), the LLVM-level
+ // builtin returns the cooperative matrix result directly.
+ llvm::FunctionType *BuiltinType =
+ llvm::FunctionType::get(ResultLLVMType, ArgTypes,
+ /*IsVarArg=*/false);
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
- "coopmat.broadcast");
+ llvm::FunctionCallee Builtin =
+ Module.getOrInsertFunction(BuiltinName, BuiltinType);
+
+ return Builder.CreateCall(Builtin, Args, "coopmat.result");
}
llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
llvm::Value *LHS,
llvm::Value *RHS,
QualType ResultTy) {
- auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
-
switch (Opcode) {
case BO_Add:
- return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_add", {LHS, RHS}, ResultTy);
case BO_Sub:
- return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub(LHS, RHS, "coopmat.isub");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_sub", {LHS, RHS}, ResultTy);
case BO_Mul:
- // mat * mat → element-wise FMul/IMul
- // mat * scalar → handled separately (see scalar path below)
+ // Matrix * scalar.
if (!RHS->getType()->isVectorTy()) {
- // mat * scalar: broadcast scalar then element-wise mul
- llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
- return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_mul", {LHS, RHS},
+ ResultTy);
}
- return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul(LHS, RHS, "coopmat.imul");
+
+ // Matrix * matrix.
+ return EmitCoopMatBuiltinCall("coop_mat_binary_mul", {LHS, RHS}, ResultTy);
case BO_Div:
- if (IsFloat)
- return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned)
- return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
- return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_div", {LHS, RHS}, ResultTy);
default:
- llvm_unreachable("Unsupported cooperative matrix binary op");
+ llvm_unreachable("Unsupported cooperative matrix binary operation");
}
-}
\ No newline at end of file
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatNeg(llvm::Value *Operand,
+ QualType ResultTy) {
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_neg", {Operand}, ResultTy);
+}
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index ff0ed1473d7df9..734a623e9e0bf3 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1092,6 +1092,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
case CK_BuiltinFnToFnPtr:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_IntToOCLSampler:
@@ -1548,6 +1549,7 @@ static bool castPreservesZero(const CastExpr *CE) {
// Language extensions.
case CK_VectorSplat:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_NonAtomicToAtomic:
case CK_AtomicToNonAtomic:
case CK_HLSLVectorTruncation:
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 30d693fb371cb6..87877bed010c14 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -618,6 +618,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index e2ec07bbae711a..5aab1dfd6ac0c4 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1388,6 +1388,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 354143adf5c028..82d814e6b0795c 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -3015,7 +3015,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
CGF.EmitIgnoredExpr(E);
return nullptr;
}
- case CK_MatrixCast: {
+ case CK_MatrixCast:
+ case CK_CoopMatrixCast: {
return EmitScalarConversion(Visit(E), E->getType(), DestTy,
CE->getExprLoc());
}
@@ -3689,15 +3690,10 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
- if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
- llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy =
- E->getType()->getAs<CooperativeMatrixType>()->getElementType();
- if (CompTy->isFloatingType())
- return Builder.CreateFNeg(Val, "coopmat.fneg");
- else
- return Builder.CreateNeg(Val, "coopmat.ineg");
- }
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType())
+ return CGF.EmitCoopMatNeg(CGF.EmitScalarExpr(E->getSubExpr()),
+ E->getType());
+
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 7b69f7ff450108..50778d662d430d 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5191,15 +5191,6 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
- /// Helper function for EmitCoopMatBinaryOp
- llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy);
-
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
- llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
- llvm::Value *RHS, QualType ResultTy);
-
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
@@ -5543,6 +5534,20 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
SourceLocation loc);
+ /// Helper function to emit coop matrix builtin call.
+ llvm::Value *EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
+ /// EmitCoopMatNeg - Emit the computation of the negate op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
+
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
/// point operation, expressed as the maximum relative error in ulp.
void SetFPAccuracy(llvm::Value *Val, float Accuracy);
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index c0ba3582076415..540dbc069c62b1 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -332,11 +332,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
+ assert(isa<ConstantMatrixType>(Ty) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
+ if (const auto *CMTy = dyn_cast<CooperativeMatrixType>(Ty))
+ return getTypeInfo(CMTy->getElementType());
+
// Enum types are distinct types. In C++ they have "underlying types",
// however they aren't related for TBAA.
if (const EnumType *ETy = dyn_cast<EnumType>(Ty)) {
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 5850604f88207b..6909370d7f1090 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -137,10 +137,8 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
case Type::CooperativeMatrix: {
- const auto *MT = cast<CooperativeMatrixType>(QT);
- return Builder.getArrayType(convertType(MT->getElementType()),
- MT->getNumRows() * MT->getNumColumns(),
- ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ llvm::reportFatalInternalError(
+ "Cooperative Matrix type not supported in ABI lowering library");
}
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index 73d1753349fd40..f6bb43482c705e 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -48,7 +48,6 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
- const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index e8d4660fd36b22..ecc7805ddd446d 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -1079,6 +1079,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
case CK_ZeroToOCLOpaqueType:
case CK_IntToOCLSampler:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
return false;
case CK_BooleanToSignedIntegral:
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index c797bdb11e0cbd..c472cbeac9ef95 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -1607,6 +1607,15 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
return TC_Success;
}
+ if (SrcType->isCooperativeMatrixType() &&
+ DestType->isCooperativeMatrixType()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind)) {
+ SrcExpr = ExprError();
+ return TC_Failed;
+ }
+ return TC_Success;
+ }
+
if (SrcType == Self.Context.AMDGPUFeaturePredicateTy &&
DestType == Self.Context.getLogicalOperationType()) {
SrcExpr = Self.AMDGPU().ExpandAMDGPUPredicateBuiltIn(SrcExpr.get());
@@ -3247,6 +3256,13 @@ void CastOperation::CheckCStyleCast() {
return;
}
+ if (DestType->getAs<CooperativeMatrixType>() ||
+ SrcType->getAs<CooperativeMatrixType>()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind))
+ SrcExpr = ExprError();
+ return;
+ }
+
if (const VectorType *DestVecTy = DestType->getAs<VectorType>()) {
if (Self.CheckAltivecInitFromScalar(OpRange, DestType, SrcType)) {
SrcExpr = ExprError();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 85b5e8aba592fb..cfabb76a2793af 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17458,7 +17458,7 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
ArgError = true;
} else {
ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
- if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
ArgError = true;
}
}
@@ -17578,7 +17578,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
M2Ty->getElementType().getUnqualifiedType())
Diag(Loc, diag::err_coop_matrix_element_type);
- if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
}
@@ -17593,7 +17593,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17715,9 +17715,8 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
- auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
- if (!ConstMType && !CoopMType) {
+ auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
+ if (!MType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17726,23 +17725,12 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- if (ConstMType) {
- QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
- if (CoopMType) {
- QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(), CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
-
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 343d260fb1a26d..67ef14deefed52 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14643,7 +14643,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
- if (!VDecl->getType()->isMatrixType()) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 33e9f21101b77d..71f11b7103492f 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5186,7 +5186,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
}
// If the base is a matrix type, try to create a new MatrixSubscriptExpr.
- if (base->getType()->isMatrixType()) {
+ if (base->getType()->isMatrixType() ||
+ base->getType()->isCooperativeMatrixType()) {
if (CheckAndReportCommaError(ArgExprs.front()))
return ExprError();
@@ -8060,16 +8061,6 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
- if (srcTy->isCooperativeMatrixType()) {
- const CooperativeMatrixType *matSrcType =
- srcTy->getAs<CooperativeMatrixType>();
- const CooperativeMatrixType *matDestType =
- destTy->getAs<CooperativeMatrixType>();
-
- return matSrcType->getNumRows() == matDestType->getNumRows() &&
- matSrcType->getNumColumns() == matDestType->getNumColumns();
- }
-
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -8077,6 +8068,22 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
+bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
+ QualType destTy) {
+ if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
+ return false;
+
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getScope() == matDestType->getScope() &&
+ matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns() &&
+ matSrcType->getUse() == matDestType->getUse();
+}
+
bool Sema::areVectorTypesSameSize(QualType SrcTy, QualType DestTy) {
assert(DestTy->isVectorType() || SrcTy->isVectorType());
@@ -8182,6 +8189,27 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
return false;
}
+bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind) {
+ if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
+ if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
+ << DestTy << SrcTy << R;
+ }
+ } else if (SrcTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << SrcTy << DestTy << R;
+ } else if (DestTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << DestTy << SrcTy << R;
+ }
+
+ Kind = CK_CoopMatrixCast;
+ return false;
+}
+
bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
CastKind &Kind) {
assert(VectorTy->isVectorType() && "Not a vector type!");
@@ -11410,6 +11438,9 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
ArithConvKind::Arithmetic);
if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
+ if (!IsDiv &&
+ (LHSTy->isCooperativeMatrixType() || RHSTy->isCooperativeMatrixType()))
+ return CheckCoopMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
@@ -11818,7 +11849,7 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -11973,7 +12004,7 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -14026,9 +14057,10 @@ QualType Sema::CheckMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
return InvalidOperands(Loc, LHS, RHS);
}
-QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
- SourceLocation Loc,
- bool IsCompAssign) {
+QualType Sema::CheckCoopMatrixElementwiseOperands(ExprResult &LHS,
+ ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
if (!IsCompAssign) {
LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
if (LHS.isInvalid())
@@ -14038,33 +14070,55 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
- if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
- auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
- auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) &&
- "At least one operand must be a matrix");
- if (LHSMatType && RHSMatType) {
- if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
- return InvalidOperands(Loc, LHS, RHS);
+ // For conversion purposes, we ignore any qualifiers.
+ // For example, "const float" and "float" are equivalent.
+ QualType LHSType = LHS.get()->getType().getUnqualifiedType();
+ QualType RHSType = RHS.get()->getType().getUnqualifiedType();
- if (Context.hasSameType(LHSMatType, RHSMatType))
- return Context.getCommonSugaredType(
- LHS.get()->getType().getUnqualifiedType(),
- RHS.get()->getType().getUnqualifiedType());
+ const CooperativeMatrixType *LHSMatType =
+ LHSType->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *RHSMatType =
+ RHSType->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a cooperative matrix");
- QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
- if (!Context.hasSameType(LHSELTy, RHSELTy))
- return InvalidOperands(Loc, LHS, RHS);
+ if (Context.hasSameType(LHSType, RHSType))
+ return Context.getCommonSugaredType(LHSType, RHSType);
- return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy),
- LHSMatType->getScope(), LHSMatType->getNumRows(),
- RHSMatType->getNumColumns(), LHSMatType->getUse());
- }
+ // Type conversion may change LHS/RHS. Keep copies to the original results, in
+ // case we have to return InvalidOperands.
+ ExprResult OriginalLHS = LHS;
+ ExprResult OriginalRHS = RHS;
+ if (LHSMatType && !RHSMatType) {
+ RHS = tryConvertExprToType(RHS.get(), LHSMatType->getElementType());
+ if (!RHS.isInvalid())
+ return LHSType;
+
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
}
+ if (!LHSMatType && RHSMatType) {
+ LHS = tryConvertExprToType(LHS.get(), RHSMatType->getElementType());
+ if (!LHS.isInvalid())
+ return RHSType;
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
+ }
+
+ return InvalidOperands(Loc, LHS, RHS);
+}
+
+QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -14090,6 +14144,43 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
}
+QualType Sema::CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ return CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+}
+
static bool isLegalBoolVectorBinaryOp(BinaryOperatorKind Opc) {
switch (Opc) {
default:
@@ -15868,7 +15959,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
// Cooperative matrix support
- if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ if (LHSTy->isCooperativeMatrixType() && RHSTy->isCooperativeMatrixType()) {
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
@@ -15876,7 +15967,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
@@ -15906,7 +15997,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
- if (!LHSExpr->getType()->isMatrixType()) {
+ if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
@@ -16576,7 +16667,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
- if (Opc == UO_Minus && Ty->isMatrixType()) {
+ if (Opc == UO_Minus && Ty->isCooperativeMatrixType()) {
return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
{InputExpr});
}
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index f44bdf45d1c913..77284c34f4ea75 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2459,9 +2459,11 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
if (!MA)
return TemplateDeductionResult::NonDeducedMismatch;
- // Check that the dimensions are the same
- if (MP->getNumRows() != MA->getNumRows() ||
- MP->getNumColumns() != MA->getNumColumns()) {
+ // Check that the dimensions, scope and use are the same
+ if (MP->getScope() != MA->getScope() ||
+ MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns() ||
+ MP->getUse() != MA->getUse()) {
return TemplateDeductionResult::NonDeducedMismatch;
}
// Perform deduction on element types.
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 476f4d24fc1ecf..9e956ad0af304d 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2527,11 +2527,9 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc, Expr *ScopeExpr,
- Expr *UseExpr, bool IsCoopMat) {
- if (!IsCoopMat)
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc) {
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2612,29 +2610,76 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
- if (IsCoopMat) {
- std::optional<llvm::APSInt> ValueScope =
- ScopeExpr->getIntegerConstantExpr(Context);
- unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
- std::optional<llvm::APSInt> ValueUse =
- UseExpr->getIntegerConstantExpr(Context);
- unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
-
- if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr)
- << ColRange << "matrix scope";
- return QualType();
- }
- if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
- return QualType();
- }
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
- }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
+QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
+ Expr *NumRows, Expr *NumCols, Expr *UseExpr,
+ SourceLocation AttrLoc) {
+ std::optional<llvm::APSInt> ValueRows =
+ NumRows->getIntegerConstantExpr(Context);
+ std::optional<llvm::APSInt> ValueColumns =
+ NumCols->getIntegerConstantExpr(Context);
+
+ auto const RowRange = NumRows->getSourceRange();
+ auto const ColRange = NumCols->getSourceRange();
+
+ // Both are row and column expressions are invalid.
+ if (!ValueRows && !ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange << ColRange;
+ return QualType();
+ }
+
+ // Only the row expression is invalid.
+ if (!ValueRows) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange;
+ return QualType();
+ }
+
+ // Only the column expression is invalid.
+ if (!ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << ColRange;
+ return QualType();
+ }
+
+ // Check the matrix dimensions.
+ unsigned MatrixRows = static_cast<unsigned>(ValueRows->getZExtValue());
+ unsigned MatrixColumns = static_cast<unsigned>(ValueColumns->getZExtValue());
+ if (MatrixRows == 0 && MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size)
+ << "matrix" << RowRange << ColRange;
+ return QualType();
+ }
+ if (MatrixRows == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << RowRange;
+ return QualType();
+ }
+ if (MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << ColRange;
+ return QualType();
+ }
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+}
+
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
if ((T->isArrayType() && !getLangOpts().allowArrayReturnTypes()) ||
T->isFunctionType()) {
@@ -6009,7 +6054,7 @@ static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
}
}
- llvm_unreachable("no matrix_type attribute found at the expected location!");
+ llvm_unreachable("no coop_mat attribute found at the expected location!");
}
namespace {
@@ -9023,34 +9068,40 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S, bool IsCoopMat = false) {
- if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
+ Sema &S) {
+ if (!S.getLangOpts().MatrixTypes) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- unsigned int NumAttrs = IsCoopMat ? 4 : 2;
- if (Attr.getNumArgs() != NumAttrs) {
+ if (Attr.getNumArgs() != 2) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << NumAttrs;
+ << Attr << 2;
return;
}
- if (IsCoopMat) {
- Expr *Scope = Attr.getArgAsExpr(0);
- Expr *RowsExpr = Attr.getArgAsExpr(1);
- Expr *ColsExpr = Attr.getArgAsExpr(2);
- Expr *Use = Attr.getArgAsExpr(3);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
- Scope, Use, /* IsCoopMat */ true);
- if (!T.isNull())
- CurType = T;
- } else {
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
+}
+
+/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
+ Sema &S) {
+ if (Attr.getNumArgs() != 4) {
+ S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+ << Attr << 4;
+ return;
}
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildCoopMatrixType(CurType, Scope, RowsExpr, ColsExpr, Use,
+ Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9335,7 +9386,7 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
break;
case ParsedAttr::AT_CoopMatrixType:
- HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ HandleCoopMatrixTypeAttr(type, attr, state.getSema());
attr.setUsedAsTypeAttr();
break;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 43adeebd945006..f4c78b965d24aa 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -30,7 +30,7 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
// ---------------------------------------------------------------------------
-// 1. Basic type construction — four use roles, float element type.
+// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
@@ -43,7 +43,7 @@ typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
// AST: TypedefDecl {{.*}} MatA_int8x8
// ---------------------------------------------------------------------------
-// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// Type printer — VarDecls carry the coop_mat attribute in their type string.
// ---------------------------------------------------------------------------
void test_type_spelling(void) {
MatA_float16x16 a;
@@ -58,17 +58,7 @@ void test_type_spelling(void) {
// AST: VarDecl {{.*}} d {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
-// float(4B)*16*16 = 1024 B = 8192 bits
-// int(4B)*8*8 = 256 B = 2048 bits
-// ---------------------------------------------------------------------------
-void test_sizeof(void) {
- _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
- _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
-}
-
-// ---------------------------------------------------------------------------
-// 4. Parameter / return type — type preserved through function boundary.
+// Parameter / return type — type preserved through function boundary.
// ---------------------------------------------------------------------------
MatA_float16x16 test_param_return(MatA_float16x16 in) {
return in;
@@ -78,7 +68,7 @@ MatA_float16x16 test_param_return(MatA_float16x16 in) {
// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 5. TypeLoc operand traversal — all four operand slots populated.
+// TypeLoc operand traversal — all four operand slots populated.
// ---------------------------------------------------------------------------
void test_typeloc_operands(void) {
float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
@@ -88,7 +78,7 @@ void test_typeloc_operands(void) {
// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// RecursiveASTVisitor — element type (half) reachable through the node.
// ---------------------------------------------------------------------------
void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
(void)x;
@@ -97,8 +87,8 @@ void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))
// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
-// same canonical type; taking a pointer across them compiles cleanly.
+// mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
@@ -109,6 +99,6 @@ void test_merge_types(void) {
}
// ---------------------------------------------------------------------------
-// 8. PCH serialisation round-trip.
+// PCH serialisation round-trip.
// ---------------------------------------------------------------------------
// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 951fedea44ee4d..dac8117f971d2a 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1912,7 +1912,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
-DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, Type)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
>From 125a9060506bbf0bd498ae2e1397d9075304b7f4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 14:31:27 -0400
Subject: [PATCH 15/38] iMinor issues with previous commit
---
clang/include/clang/AST/TypeLoc.h | 2 +-
clang/lib/CodeGen/CGExpr.cpp | 2 +-
clang/lib/Sema/SemaChecking.cpp | 3 +--
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 122090f7e6ec57..b7432d03a92dd7 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2217,7 +2217,7 @@ class CooperativeMatrixTypeLoc
public:
/// The location of the attribute name, i.e.
/// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^^~~~~~~~~~~~~~~~~~~
+ /// ^~~~~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index ecd89bd18a0db8..8f4f8317fd7971 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -6417,6 +6417,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
@@ -7702,7 +7703,6 @@ CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
assert(!Args.empty() &&
"Expected at least one argument for cooperative matrix builtin");
- llvm::LLVMContext &Context = CGM.getLLVMContext();
llvm::Module &Module = CGM.getModule();
llvm::Type *ResultLLVMType = ConvertType(ResultTy);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index cfabb76a2793af..0fed3d8253a8db 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17726,8 +17726,7 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
>From 2f0fe37a9eb9149382defd2c1d199efd2e78a229 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 15:35:07 -0400
Subject: [PATCH 16/38] Add regression test for change in SPIRVUtils.cpp
---
.../SPIRV/coop-matrix-builtin-demangle.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
diff --git a/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
new file mode 100644
index 00000000000000..e99d2199b7131e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=spirv64-unknown-unknown \
+; RUN: -spirv-ext=+SPV_KHR_cooperative_matrix \
+; RUN: -filetype=asm < %s | FileCheck %s
+
+target triple = "spirv64-unknown-unknown"
+
+%coopmat.f32.sc3.16x16.u0 =
+ type target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+
+; ---------------------------------------------------------------------------
+; Suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_mangled_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_mangled_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
+; ---------------------------------------------------------------------------
+; No-suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_canonical_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_canonical_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
>From f1c24cdf1e82cfd2b4ea784a9eb50c25522d2e58 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 17:00:22 -0400
Subject: [PATCH 17/38] Address review comments
---
clang/include/clang/AST/TypeBase.h | 19 ++------
clang/include/clang/AST/TypeProperties.td | 6 +--
.../include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/lib/AST/ASTStructuralEquivalence.cpp | 2 +-
clang/lib/Headers/opencl-c-base.h | 47 ++++++++++---------
5 files changed, 33 insertions(+), 43 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 0b83ac3fd03e40..5328e2db64d2c4 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4603,13 +4603,8 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
- : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
- NumRows, NumColumns, Use, CanonicalType) {}
-
- CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
- unsigned Scope, unsigned NumRows, unsigned NumColumns,
- unsigned Use, QualType CanonicalType)
- : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ : Type(Type::CooperativeMatrix, CanonicalType,
+ ElementType->getDependence()),
ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
Scope(Scope), Use(Use) {}
@@ -4629,12 +4624,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into
- /// a vector representation.
- unsigned getNumElementsFlattened() const {
- return getNumRows() * getNumColumns();
- }
-
/// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
@@ -4658,12 +4647,12 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
}
void Profile(llvm::FoldingSetNodeID &ID) {
- Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
}
static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
- unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
unsigned Use, TypeClass TypeClass) {
ID.AddPointer(ElementType.getAsOpaquePtr());
ID.AddInteger(NumRows);
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index d134edc39e92ce..1d486ba3c20587 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -267,15 +267,15 @@ let Class = CooperativeMatrixType in {
def : Property<"elementType", QualType> {
let Read = [{ node->getElementType() }];
}
- def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
let Read = [{ node->getNumColumns() }];
}
- def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+ def : Property<"use", UInt32> { let Read = [{ node->getUse() }]; }
def : Creator<[{
- return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ return ctx.getCooperativeMatrixType(elementType, scope, numRows, numColumns, use);
}]>;
}
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 8a583ad504c706..56ce32cf3e7c5d 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 8ec1100f9f4d89..fd4f55e43f665d 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1155,7 +1155,7 @@ bool ASTStructuralEquivalence::isEquivalent(
const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
// The element types must be structurally equivalent and the number of rows
- // and columns must match.
+ // and columns and scope and use must match.
if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
Mat2->getElementType()) ||
Mat1->getScope() != Mat2->getScope() ||
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 7b3d239299c6e7..83bb35f1d886af 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,6 +632,30 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
+
#ifdef cl_intel_device_side_avc_motion_estimation
#define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0
@@ -771,27 +795,4 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
-typedef enum coop_matrix_operands_t {
- CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
- CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
-} coop_matrix_operands_t;
-
-typedef enum coop_matrix_use_t {
- CLK_COOPERATIVE_MATRIX_A = 0,
- CLK_COOPERATIVE_MATRIX_B = 1,
- CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
-} coop_matrix_use_t;
-
-typedef enum coop_matrix_layout_t {
- CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
- CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
-} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From d9e7ccc3940ada259dab7b26bf378697f626e87d Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sat, 12 Sep 2026 17:21:24 -0400
Subject: [PATCH 18/38] Add cooperative matrix type support in missing sites
---
clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp | 1 +
clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp | 1 +
clang/lib/CIR/CodeGen/CIRGenFunction.cpp | 2 ++
clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 2 ++
4 files changed, 6 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 91535c57132bd0..845e8c9810dd3e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -536,6 +536,7 @@ mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index cfe575f4670dd4..94a17748e88a14 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -648,6 +648,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLArrayRValue:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index e5e83a26470181..c7f97b344ccd83 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -82,6 +82,7 @@ cir::TypeEvaluationKind CIRGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -1681,6 +1682,7 @@ void CIRGenFunction::emitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index 763ed026f87146..ae154e00752ddd 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -902,6 +902,7 @@ const char *vTableClassNameForType(const CIRGenModule &cgm, const Type *ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -1375,6 +1376,7 @@ mlir::Attribute CIRGenItaniumRTTIBuilder::buildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
>From 3c41a0a239aead9676ac1c91dc671660d29d853b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 17 Sep 2026 21:44:28 -0400
Subject: [PATCH 19/38] Add cooperative matrix related changes to CIR
---
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 14 ++++++++++++++
clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 9 +++++++++
2 files changed, 23 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 97ce3fbcabcbce..f7202caf8e5f30 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -2330,6 +2330,20 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
case Builtin::BI__builtin_masked_store:
case Builtin::BI__builtin_masked_compress_store:
case Builtin::BI__builtin_masked_scatter:
+ case Builtin::BIcoop_mat_load:
+ case Builtin::BIcoop_mat_store:
+ case Builtin::BIcoop_mat_mulAdd:
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ case Builtin::BIcoop_mat_scalar_mul:
+ case Builtin::BIcoop_mat_scalar_neg:
+ case Builtin::BIcoop_mat_init:
+ case Builtin::BIcoop_mat_length:
+ case Builtin::BI__builtin_matrix_transpose:
+ case Builtin::BI__builtin_matrix_column_major_load:
+ case Builtin::BI__builtin_matrix_column_major_store:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 836f27cf5345ef..f65e94ffa88c12 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -872,6 +872,10 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
}
mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
+ if (e->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ cgf.cgm.errorNYI(e->getSourceRange(), "cooperative matrix unary minus");
+ return {};
+ }
ignoreResultAssign = false;
mlir::Value operand;
if (!promotionType.isNull())
@@ -1236,6 +1240,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
mlir::Value VisitBin##OP(const BinaryOperator *e) { \
+ if (e->getType()->isCooperativeMatrixType()) { \
+ cgf.cgm.errorNYI(e->getSourceRange(), \
+ "cooperative matrix binary operator"); \
+ return {}; \
+ } \
QualType promotionTy = getPromotionType(e->getType()); \
auto result = emit##OP(emitBinOps(e, promotionTy)); \
if (result && !promotionTy.isNull()) \
>From 613f6c883501727e57e3992443cd2b3d40263203 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 17 Sep 2026 23:45:47 -0400
Subject: [PATCH 20/38] Address review comments: 1. Update comments to remove
function names 2. Add change to emit error if extension is not enabled using
'pragma OPENCL_EXTENSION ... enable' in OpenCL source
---
.../include/clang/Basic/DiagnosticSemaKinds.td | 3 +++
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/include/clang/Sema/Sema.h | 9 ++++-----
clang/lib/AST/ASTContext.cpp | 3 +--
clang/lib/CodeGen/CodeGenFunction.h | 6 ++----
clang/lib/Sema/SemaType.cpp | 18 +++++++++++++++---
.../CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 ++
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 ++
.../coop-mat-init-length-codegen.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 ++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 7 +++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 2 ++
14 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 300fda4bed280c..c80d5f2222a5fb 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11989,6 +11989,9 @@ def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
// OpenCL C Cooperative Matrix.
+def ext_coopmat_without_pragma
+ : Error<"cooperative matrix types require OpenCL extension '%0' to be enabled via pragma">;
+
def err_invalid_coopmat_attr
: Error<"invalid argument of cooperative matrix attribute">;
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 56ce32cf3e7c5d..8a583ad504c706 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 6853a2fd2e8698..6da59918d8854f 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8092,10 +8092,10 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
- // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
- // We allow casting between cooperative matrixes of the same scope, use, and
- // same dimensions i.e. when they have the same number of rows and columns.
- // Returns true if the cast is invalid.
+ /// Check type constraints for cooperative matrix casts. We allow casting
+ /// between cooperative matrixes of the same scope, use, and same dimensions
+ /// i.e. when they have the same number of rows and columns. Returns true if
+ /// the cast is invalid.
bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
@@ -15415,7 +15415,6 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
-
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
SourceLocation AttrLoc);
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 0317ce1268150a..c1086d2ed5e4e3 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -10815,8 +10815,7 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
-/// matrix types are compatible.
+/// Return true if the two specified cooperative matrix types are compatible.
static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 50778d662d430d..e689a751b7a9ca 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5539,13 +5539,11 @@ class CodeGenFunction : public CodeGenTypeCache {
llvm::ArrayRef<llvm::Value *> Args,
QualType ResultTy);
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
+ /// Emit the computation of the specified binary op, returning the result.
llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
llvm::Value *RHS, QualType ResultTy);
- /// EmitCoopMatNeg - Emit the computation of the negate op,
- /// returning the result.
+ /// Emit the computation of the negate op, returning the result.
llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 9e956ad0af304d..2f2b16c122b447 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2676,8 +2676,19 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
return QualType();
}
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
+ auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ if (getLangOpts().OpenCL) {
+ if (!getOpenCLOptions().isSupported("cl_khr_cooperative_matrix",
+ getLangOpts()))
+ Diag(AttrLoc, diag::err_opencl_requires_extension)
+ << 0 << Result << "cl_khr_cooperative_matrix";
+ else if (!getOpenCLOptions().isAvailableOption("cl_khr_cooperative_matrix",
+ getLangOpts()))
+ Diag(AttrLoc, diag::ext_coopmat_without_pragma)
+ << "cl_khr_cooperative_matrix";
+ }
+ return Result;
}
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
@@ -9079,6 +9090,7 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
<< Attr << 2;
return;
}
+
Expr *RowsExpr = Attr.getArgAsExpr(0);
Expr *ColsExpr = Attr.getArgAsExpr(1);
QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
@@ -9086,7 +9098,7 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
CurType = T;
}
-/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+/// "coop_mat" attribute, like ext_vector_type
static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Sema &S) {
if (Attr.getNumArgs() != 4) {
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9d55bd8c61cc3c..c070010c24b499 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -19,6 +19,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Matrix type aliases used across tests
// ---------------------------------------------------------------------------
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index ad8df2939269f5..3dada9b4ef5de8 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -14,6 +14,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index dfa763b060e720..0b7a825b5d369c 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -5,6 +5,8 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Type definitions.
// ---------------------------------------------------------------------------
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index e8fad815806ece..38d98b28482bfe 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -33,6 +33,8 @@
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index 799f7a9e15a352..b32f6549deba50 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -15,6 +15,8 @@
// ---------------------------------------------------------------------------
// Type definitions shared across all tests.
// ---------------------------------------------------------------------------
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 0c27978d492a92..351728d2454943 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -12,6 +12,13 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+// ---------------------------------------------------------------------------
+// 0. Use of coop mat type without specifying pragma
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expected-error {{cooperative matrix types require OpenCL extension 'cl_khr_cooperative_matrix' to be enabled via pragma}}
+
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index bcedd94fdbf17d..dafa5b3bf123af 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -16,6 +16,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index f4c78b965d24aa..3f8b221c3edeb0 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -29,6 +29,8 @@
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
>From dd739ca3b5c1cc89e63e59de64a6fac1583de3d4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 18 Sep 2026 00:21:21 -0400
Subject: [PATCH 21/38] Address review comments:
1. Fix Matrix Scalar multiply for cases where LHS is scalar
2. Use getBuiltinID() to check for coop mat builtins instead of name
---
clang/lib/Sema/SemaDecl.cpp | 16 ++++++++++++++--
clang/lib/Sema/SemaExpr.cpp | 4 +++-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 4 ++++
3 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 67ef14deefed52..0e05bdb8729c6e 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14114,9 +14114,21 @@ bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
if (!Fname)
return false;
- if (Fname->getName().starts_with("coop_mat") &&
- !Fname->getName().starts_with("coop_mat_length"))
+ switch (F->getBuiltinID()) {
+ case Builtin::BIcoop_mat_load:
+ case Builtin::BIcoop_mat_store:
+ case Builtin::BIcoop_mat_mulAdd:
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ case Builtin::BIcoop_mat_scalar_mul:
+ case Builtin::BIcoop_mat_scalar_neg:
+ case Builtin::BIcoop_mat_init:
return true;
+ default:
+ return false;
+ }
return false;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 71f11b7103492f..872df233400bac 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -15967,8 +15967,10 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType()) {
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (RHSTy->isCooperativeMatrixType() && LHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, RHSExpr, LHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 3dada9b4ef5de8..28f7660957a624 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -79,9 +79,13 @@ kernel void test_scalar_mul(__global float *ptr, float s) {
a = coop_mat_load(ptr, ROW_MAJOR, 16);
r = a * s;
(void)r;
+ MatA_t r2;
+ r2 = s * a;
+ (void)r2;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
>From caf55448a5fde647e7cdf8278d633103dd97de48 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 18 Sep 2026 02:24:09 -0400
Subject: [PATCH 22/38] Address more review comments: 1. Add documentation for
'coop_mat(...)' attribute 2. Add more checks for scope, use, and element type
3. Fix some typos
---
clang/include/clang/Basic/Attr.td | 2 +-
clang/include/clang/Basic/AttrDocs.td | 47 +++++++++++++++++++
.../clang/Basic/DiagnosticSemaKinds.td | 5 +-
clang/lib/Sema/SemaChecking.cpp | 15 +++---
clang/lib/Sema/SemaType.cpp | 22 ++++++++-
clang/test/AST/undocumented-attrs.cpp | 3 +-
6 files changed, 81 insertions(+), 13 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index f66fc29fdac450..14c0452bc0a15e 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3986,7 +3986,7 @@ def CoopMatrixType : TypeAttr {
let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
- let Documentation = [Undocumented];
+ let Documentation = [CoopMatrixTypeDocs];
}
def Visibility : InheritableAttr {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 0ab1429ff09d67..93174cae1d5391 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1479,6 +1479,53 @@ See the OpenCL documentation for some more complete examples.
}];
}
+def CoopMatrixTypeDocs : Documentation {
+ let Category = DocCatType;
+ let Heading = "coop_mat";
+ let Content = [{
+The `coop_mat` attribute declares an OpenCL cooperative matrix type. Cooperative
+matrix operations are distributed across a set of invocations, such as a
+subgroup, allowing the implementation to optimize matrix multiply operations.
+
+The attribute is applied to a scalar numerical component type and takes four
+arguments:
+
+```c
+__attribute__((coop_mat(scope, rows, columns, use)))
+```
+- scope specifies the memory scope of the cooperative matrix.
+ It must be one of the supported OpenCL memory scopes, such as CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP.
+- rows specifies the number of rows in the matrix. It must be a constant expression.
+- columns specifies the number of columns in the matrix. It must be a constant expression.
+- use specifies the operand role of the matrix in coop_mat_muladd. It must be one of:
+ CLK_COOPERATIVE_MATRIX_A
+ CLK_COOPERATIVE_MATRIX_B
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+The coop_mat attribute can be used to declare local variables, function parameters, and function return values. Cooperative matrix types must not be used as kernel arguments.
+
+```c
+float vmat1 __attribute__((coop_mat(
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 8,
+ 8,
+ CLK_COOPERATIVE_MATRIX_A)));
+
+typedef half chmat44 __attribute__((coop_mat(
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 4,
+ 4,
+ CLK_COOPERATIVE_MATRIX_B)));
+chmat44 vmat2;
+```
+The base component type must be a scalar numerical type.
+Implicit and explicit casts between cooperative matrix types are not generally permitted.
+Conversion between cooperative matrix types is allowed only when the types have identical scope, dimensions, and use parameters, and their component types are compatible according to the OpenCL C conversion rules.
+Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_matmul_add, and coop_mat_length.
+See the OpenCL documentation for some more complete examples.
+ }];
+}
+
def DiagnoseIfDocs : Documentation {
let Category = DocCatFunction;
let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index c80d5f2222a5fb..c696c0e6a9252e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12027,9 +12027,12 @@ def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
"CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
-def err_coop_matrix_element_type
+def err_mismatched_coop_matrix_element_type
: Error<"inconsistent cooperative matrix element type">;
+def err_invalid_coop_matrix_element_type
+ : Error<"invalid cooperative matrix element type">;
+
def err_coop_matrix_row_or_col_mismatch
: Error<"mismatch of cooperative matrix row or column">;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 0fed3d8253a8db..0ce7084df1f898 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17541,7 +17541,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
- if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
Arg0->getBeginLoc());
@@ -17576,7 +17576,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
if (MOutTy->getElementType().getUnqualifiedType() !=
M2Ty->getElementType().getUnqualifiedType())
- Diag(Loc, diag::err_coop_matrix_element_type);
+ Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
@@ -17605,7 +17605,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType()) {
- Diag(ALoc, diag::err_coop_matrix_element_type);
+ Diag(ALoc, diag::err_mismatched_coop_matrix_element_type);
return true;
}
return false;
@@ -17619,8 +17619,9 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
- CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ if (CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(),
+ Arg1->getType(), Arg1->getBeginLoc()))
+ return ExprError();
TheCall->setType(Arg0->getType());
@@ -17668,11 +17669,11 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType())
- return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
- return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+ return ExprError(Diag(Loc1, diag::err_mismatched_coop_matrix_element_type));
if (M0Ty->getNumRows() != M2Ty->getNumRows())
return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 2f2b16c122b447..8bc9451365c06e 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2663,17 +2663,35 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
}
std::optional<llvm::APSInt> ValueScope =
ScopeExpr->getIntegerConstantExpr(Context);
+ // Only the scope expression is invalid.
+ if (!ValueScope) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant
+ << ScopeExpr->getSourceRange();
+ return QualType();
+ }
unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
std::optional<llvm::APSInt> ValueUse =
UseExpr->getIntegerConstantExpr(Context);
+ // Only the use expression is invalid.
+ if (!ValueUse) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant
+ << UseExpr->getSourceRange();
+ return QualType();
+ }
unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Scope << "matrix scope";
return QualType();
}
if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Use << "matrix use";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
+ Diag(AttrLoc, diag::err_invalid_coop_matrix_element_type) << ElementTy;
return QualType();
}
auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index dff9d3845062a2..ed82ff3f517fdd 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,7 +27,6 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
-CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -89,4 +88,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 83
+CHECK-NEXT: Total: 82
\ No newline at end of file
>From fa13584ed49cca91dbbd99391e9a6325730279fe Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 21 Sep 2026 12:24:29 -0400
Subject: [PATCH 23/38] Add missing checks for coop_mat builtins in
SemaChecking.cpp
1. Stride check in load/store
2. Check for optional matrix operand in muladd
---
.../clang/Basic/DiagnosticSemaKinds.td | 7 ++++++
clang/include/clang/Sema/Sema.h | 1 +
clang/lib/Sema/SemaChecking.cpp | 24 ++++++++++++++++++-
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index c696c0e6a9252e..6612665163b67c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12007,6 +12007,10 @@ def err_coop_mem_layout_enum
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
"CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+def err_coop_mat_stride_type
+ : Error<"stride argument for load/store of cooperative matrix must have "
+ "type 'size_t'">;
+
def err_coop_element_and_pointer_type
: Error<"inconsistent between cooperative matrix element type and buffer "
"pointer type">;
@@ -12014,6 +12018,9 @@ def err_coop_element_and_pointer_type
def err_coop_matrix_arg
: Error<"argument must be a valid cooperative matrix type">;
+def err_coop_matrix_operands_type
+ : Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
+
def err_coop_matrix_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 6da59918d8854f..7d00a598ea2e70 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3120,6 +3120,7 @@ class Sema final : public SemaBase {
SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ bool CheckCoopMatrixLoadStoreStride(Expr *Stride);
ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 0ce7084df1f898..4b39ef832298d0 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17524,6 +17524,14 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
+bool Sema::CheckCoopMatrixLoadStoreStride(Expr *Stride) {
+ if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
+ Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
+ return true;
+ }
+ return false;
+}
+
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
@@ -17532,6 +17540,8 @@ ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
return ExprError();
+ if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(2)))
+ return ExprError();
return CallResult;
}
@@ -17547,6 +17557,8 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
+ if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(3)))
+ return ExprError();
return CallResult;
}
@@ -17638,9 +17650,19 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
ExprResult CallResult) {
- if (checkArgCount(TheCall, 3))
+ if (checkArgCountRange(TheCall, 3, 4))
return ExprError();
+ // The fourth argument is optional. If present, it must be an enum.
+ if (TheCall->getNumArgs() == 4) {
+ Expr *Operands = TheCall->getArg(3);
+ QualType OperandsTy = Operands->getType().getCanonicalType();
+
+ if (!OperandsTy->isEnumeralType()) {
+ Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ }
+ }
+
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
Expr *Arg2 = TheCall->getArg(2);
>From 09de00f2c664559911c32d612e9eb35777827cb4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 21 Sep 2026 13:45:20 -0400
Subject: [PATCH 24/38] Merge type lowering for cooperative matrix between
CGBuiltin.cpp and CodeGenTypes.cpp
---
clang/lib/CodeGen/CGBuiltin.cpp | 49 +++++++++++++-----------------
clang/lib/CodeGen/CodeGenTypes.cpp | 46 +++++++++++-----------------
clang/lib/CodeGen/CodeGenTypes.h | 5 +++
3 files changed, 43 insertions(+), 57 deletions(-)
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index b6d79bd240f065..c3c18e616218b7 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2789,20 +2789,13 @@ static std::string getSPIRVBuiltinName(unsigned BuiltinID,
return "";
}
-static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
- CodeGenModule &CGM,
+static llvm::TargetExtType *getTargetExtType(CodeGenModule &CGM,
const CooperativeMatrixType *MTy) {
- llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
- // Type arguments for TargetExtType
- llvm::Type *Tys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
- // Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *RetType = llvm::TargetExtType::get(
- CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
- return RetType;
+ llvm::Type *Ty = CGM.getTypes().ConvertTypeForCoopMat(MTy);
+ auto *TargetTy = llvm::dyn_cast<llvm::TargetExtType>(Ty);
+ if (!TargetTy)
+ return nullptr;
+ return TargetTy;
}
/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
@@ -4737,7 +4730,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Layout = EmitScalarExpr(E->getArg(1));
auto Stride = EmitScalarExpr(E->getArg(2));
// Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+ llvm::TargetExtType *RetType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
@@ -4773,7 +4766,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Layout = EmitScalarExpr(E->getArg(2));
auto Stride = EmitScalarExpr(E->getArg(3));
// Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ llvm::TargetExtType *ArgType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
@@ -4817,10 +4810,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
- auto *BType = getTargetExtType(*this, CGM, MBTy);
- auto *CType = getTargetExtType(*this, CGM, MCTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
+ auto *BType = getTargetExtType(CGM, MBTy);
+ auto *CType = getTargetExtType(CGM, MCTy);
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
@@ -4848,9 +4841,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
- auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
+ auto *BType = getTargetExtType(CGM, MBTy);
llvm::Type *ElTy = ConvertType(MTy->getElementType());
// Check if the data is signed/unsigned
QualType QT = MTy->getElementType();
@@ -4884,8 +4877,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg1 = EmitScalarExpr(E->getArg(1));
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
auto *BType = Arg1->getType();
// Check if the data is signed/unsigned
@@ -4915,8 +4908,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg0 = EmitScalarExpr(E->getArg(0));
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
// Check if the data is signed/unsigned
QualType QT = MTy->getElementType();
@@ -4944,7 +4937,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_init: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
auto Init = EmitScalarExpr(E->getArg(0));
- auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
@@ -4962,7 +4955,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_length: {
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 6ac24c6554922e..c5e4017b268f8d 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -90,6 +90,20 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
Ty->setName(OS.str());
}
+llvm::Type *CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
+ llvm::Type *ElementTy = ConvertType(CMT->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElementTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {CMT->getScope(), CMT->getNumRows(), CMT->getNumColumns(),
+ CMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
@@ -125,21 +139,8 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
- if (T->isCooperativeMatrixType()) {
- const CooperativeMatrixType *DMT =
- cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
- llvm::Type *ElTy = ConvertType(DMT->getElementType());
- // Type argument for TargetExtType
- llvm::Type *ArgTys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
- DMT->getUse()};
- // Create a TargetExtType to represent a Coop matrix type
- llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
- getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
- return CoopMatType;
- }
+ if (const CooperativeMatrixType *CMT = T->getAs<CooperativeMatrixType>())
+ return ConvertTypeForCoopMat(CMT);
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -719,20 +720,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
break;
}
case Type::CooperativeMatrix: {
- const CooperativeMatrixType *DMT =
- cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
- llvm::Type *ElTy = ConvertType(DMT->getElementType());
- // Type argument for TargetExtType
- llvm::Type *ArgTys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
- DMT->getUse()};
- // Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
- getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
-
- ResultType = CoopMatType;
+ ResultType = ConvertTypeForCoopMat(T->getAs<CooperativeMatrixType>());
break;
}
case Type::FunctionNoProto:
diff --git a/clang/lib/CodeGen/CodeGenTypes.h b/clang/lib/CodeGen/CodeGenTypes.h
index 860f1f1ccdce28..dfe0df716bbf39 100644
--- a/clang/lib/CodeGen/CodeGenTypes.h
+++ b/clang/lib/CodeGen/CodeGenTypes.h
@@ -126,6 +126,11 @@ class CodeGenTypes {
/// ConvertType - Convert type T into a llvm::Type.
llvm::Type *ConvertType(QualType T);
+ /// Convert a Clang cooperative matrix type to the corresponding LLVM type.
+ /// The element type and cooperative matrix properties, including scope,
+ /// dimensions, and use, are preserved in the resulting LLVM type.
+ llvm::Type *ConvertTypeForCoopMat(const CooperativeMatrixType *CMT);
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
>From 35968f2b75979d094a6ed083a373b3385db154ed Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 22 Sep 2026 12:22:50 -0400
Subject: [PATCH 25/38] Fix clang format issue
---
clang/lib/CodeGen/CodeGenTypes.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index c5e4017b268f8d..2147f06763ecba 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -90,7 +90,8 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
Ty->setName(OS.str());
}
-llvm::Type *CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
+llvm::Type *
+CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
llvm::Type *ElementTy = ConvertType(CMT->getElementType());
// Type arguments for TargetExtType
llvm::Type *Tys[] = {ElementTy};
>From 8bee2443a47ba081b86ccb00e5b17052250a791f Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 23 Sep 2026 16:56:39 -0400
Subject: [PATCH 26/38] Add check for disallowing cooperative matrix type for
kernel parameters and update tests
---
clang/lib/Sema/SemaDecl.cpp | 3 +++
clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 10 ++++++++--
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 4 +++-
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 7 ++++++-
5 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 0e05bdb8729c6e..19620ebd70da45 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -9868,6 +9868,9 @@ static bool isOpenCLSizeDependentType(ASTContext &C, QualType Ty) {
}
static OpenCLParamType getOpenCLKernelParameterType(Sema &S, QualType PT) {
+ if (PT->isCooperativeMatrixType())
+ return InvalidKernelParam;
+
if (PT->isDependentType())
return InvalidKernelParam;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index c070010c24b499..3ec349af595769 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -126,15 +126,21 @@ kernel void test_load_local(__local float *ptr) {
// Same matrix type, __global vs __local -> two distinct symbols.
// ===========================================================================
-kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+kernel void test_store_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_init(0.0f);
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
// CHECK-SAME: ptr addrspace(1)
-kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+kernel void test_store_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_init(0.0f);
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 28f7660957a624..112a4d58edab43 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -38,8 +38,10 @@ kernel void test_load(__global float *ptr) {
// ---------------------------------------------------------------------------
// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
-kernel void test_store(__global float *ptr, MatA_t a) {
+kernel void test_store(__global float *ptr) {
+ MatA_t a;
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index 38d98b28482bfe..a7737725e5dc70 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -41,7 +41,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
// that contains the vendor-extended "coop_mat" marker.
-kernel void test_mangling(MatA_t a) { (void)a; }
+void test_mangling(MatA_t a) { (void)a; }
// MANGLE: @{{.*}}test_mangling{{.*}}(
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 351728d2454943..9e7c992a63a255 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -19,7 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expecte
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t; // expected-note {{declared here}}
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
@@ -65,3 +65,8 @@ kernel void test_bad_assignment(__global float *ptr) {
bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
(void)bad;
}
+
+// ---------------------------------------------------------------------------
+// 5. Kernel parameter cannot be of cooperative matrix type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_kernel_param(MatA_t a) {} // expected-error {{cannot be used as the type of a kernel parameter}}
>From 8718558f12acd243a069c0e0ab26261a5f74050a Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 23 Sep 2026 19:51:06 -0400
Subject: [PATCH 27/38] Update IsCoopMatBuiltin
---
clang/include/clang/Sema/Sema.h | 2 +-
clang/lib/Sema/SemaDecl.cpp | 7 +++----
clang/lib/Sema/SemaExpr.cpp | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
4 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 7d00a598ea2e70..2b12d11e9bb323 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15444,7 +15444,7 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
- bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+ bool BuiltinReturnsCoopMatrix(Expr *RHSExpr);
/// Build a function type.
///
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 19620ebd70da45..12670978778aff 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14105,8 +14105,8 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if RHSExpr is a cooperative matrix builtin call.
-bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+// Return true if cooperative matrix builtin call returns cooperative matrix.
+bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
auto call = dyn_cast<CallExpr>(RHSExpr);
if (!call)
return false;
@@ -14119,7 +14119,6 @@ bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
return false;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
- case Builtin::BIcoop_mat_store:
case Builtin::BIcoop_mat_mulAdd:
case Builtin::BIcoop_mat_binary_add:
case Builtin::BIcoop_mat_binary_sub:
@@ -14657,7 +14656,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 872df233400bac..6ac28d166042bc 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -15998,7 +15998,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
- if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(RHSExpr)) {
if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index dafa5b3bf123af..5de358100a5606 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -25,7 +25,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
// match the LHS variable type. Declare first, then assign so that
-// Sema's IsCoopMatrixBuiltin path fires correctly.
+// Sema's BuiltinReturnsCoopMatrix path fires correctly.
// ---------------------------------------------------------------------------
kernel void test_load_store(__global float *ptr,
__global float *out_ptr) {
>From 82bed6f498a853c2f2f8e13525af839bca80d122 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 16:45:02 -0400
Subject: [PATCH 28/38] Remove dimension check for cooperative matrix
---
clang/include/clang/AST/TypeBase.h | 12 ------------
clang/lib/AST/ASTContext.cpp | 3 ---
2 files changed, 15 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 5328e2db64d2c4..b3437f3e7dae67 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4598,8 +4598,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
unsigned Scope;
unsigned Use;
- static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
-
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
@@ -4624,11 +4622,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns true if \p NumElements is a valid cooperative matrix dimension.
- static constexpr bool isDimensionValid(size_t NumElements) {
- return NumElements > 0 && NumElements <= MaxElementsPerDimension;
- }
-
/// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
@@ -4641,11 +4634,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum valid number of elements per dimension.
- static constexpr unsigned getMaxElementsPerDimension() {
- return MaxElementsPerDimension;
- }
-
void Profile(llvm::FoldingSetNodeID &ID) {
Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index c1086d2ed5e4e3..4618311a0cba8c 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4951,9 +4951,6 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
"need a valid element type");
- assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
- CooperativeMatrixType::isDimensionValid(NumColumns) &&
- "need valid matrix dimensions");
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
>From 9ea10e594f30d4303c5756ccd83213d1d1c4f128 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 17:22:16 -0400
Subject: [PATCH 29/38] Update support for 'scope'
---
clang/include/clang/AST/TypeBase.h | 2 +-
clang/include/clang/Basic/AttrDocs.td | 6 ++--
clang/lib/Headers/opencl-c-base.h | 4 ---
.../coop-mat-codegen-mangling.cl | 32 +++++++++----------
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../coop-mat-init-length-codegen.cl | 6 ++--
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 +-
.../SemaOpenCL/coop-mat-init-length-sema.cl | 8 ++---
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 4 +--
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 2 +-
11 files changed, 33 insertions(+), 37 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index b3437f3e7dae67..957149d715fd39 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4624,7 +4624,7 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+ return Scope == 4; // memory_scope_sub_group
}
/// Returns true if \p Use is a valid cooperative matrix use.
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 93174cae1d5391..9023b2f594ebb3 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1494,7 +1494,7 @@ arguments:
__attribute__((coop_mat(scope, rows, columns, use)))
```
- scope specifies the memory scope of the cooperative matrix.
- It must be one of the supported OpenCL memory scopes, such as CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP.
+ Only one of the OpenCL memory scopes (memory_scope_sub_group) is supported for cooperative matrix.
- rows specifies the number of rows in the matrix. It must be a constant expression.
- columns specifies the number of columns in the matrix. It must be a constant expression.
- use specifies the operand role of the matrix in coop_mat_muladd. It must be one of:
@@ -1506,13 +1506,13 @@ The coop_mat attribute can be used to declare local variables, function paramete
```c
float vmat1 __attribute__((coop_mat(
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ memory_scope_sub_group,
8,
8,
CLK_COOPERATIVE_MATRIX_A)));
typedef half chmat44 __attribute__((coop_mat(
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ memory_scope_sub_group,
4,
4,
CLK_COOPERATIVE_MATRIX_B)));
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 83bb35f1d886af..c4263b78652559 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,10 +632,6 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
typedef enum coop_matrix_operands_t {
CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 3ec349af595769..2446e41d4aec8d 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -13,7 +13,7 @@
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
@@ -165,18 +165,18 @@ kernel void test_no_cross_contamination(__global float *gptr,
// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
// Verify distinct declare lines exist (these appear at end of IR).
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc{{[0-9]+}}_16x16_u0_f32_sc{{[0-9]+}}_16x16_u1_f32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc{{[0-9]+}}_8x32_u0_f32_sc{{[0-9]+}}_32x8_u1_f32_sc{{[0-9]+}}_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc{{[0-9]+}}_16x16_u0_i32_sc{{[0-9]+}}_16x16_u1_i32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 112a4d58edab43..0bcb3d1d2ee378 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -8,7 +8,7 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 0b7a825b5d369c..ef61b8b40fbbf2 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -10,13 +10,13 @@
// ---------------------------------------------------------------------------
// Type definitions.
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
-typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index a7737725e5dc70..f16682934759b4 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -28,7 +28,7 @@
// expected-no-diagnostics
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index b32f6549deba50..a9ea17abb27308 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -17,16 +17,16 @@
// ---------------------------------------------------------------------------
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_B))) MatB_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
-typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 9e7c992a63a255..9315f3cfeb9537 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -6,7 +6,7 @@
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
@@ -24,7 +24,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
// ---------------------------------------------------------------------------
-// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// 1. Invalid scope value (0 is not memory_scope_sub_group)
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index 5de358100a5606..7bf1f5d2ae184c 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -10,7 +10,7 @@
// expected-no-diagnostics
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 3f8b221c3edeb0..7f7daab56ebcff 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -24,7 +24,7 @@
// ---------------------------------------------------------------------------
// Enum constants (from opencl-c-base.h via -finclude-default-header)
// ---------------------------------------------------------------------------
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
>From 9398a7effc1c8023cdea750df8fb2e77add29eec Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 20:39:49 -0400
Subject: [PATCH 30/38] Update Semantic analysis to add missing checks
---
.../clang/Basic/DiagnosticSemaKinds.td | 11 +--
clang/include/clang/Sema/Sema.h | 10 +-
clang/lib/Sema/SemaChecking.cpp | 93 ++++++++++++++-----
clang/lib/Sema/SemaDecl.cpp | 14 +--
clang/lib/Sema/SemaExpr.cpp | 19 ++--
5 files changed, 90 insertions(+), 57 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6612665163b67c..694db6e5726f89 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11995,13 +11995,6 @@ def ext_coopmat_without_pragma
def err_invalid_coopmat_attr
: Error<"invalid argument of cooperative matrix attribute">;
-def err_coop_mat_not_primitive
- : Error<"cooperative matrix attribute only supported for integer and "
- "floating-point types">;
-
-def err_coop_mat_rol_col_size
- : Error<"cooperative matrix row and column must have power-of-two size">;
-
def err_coop_mem_layout_enum
: Error<"memory layout for cooperative matrix should be "
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
@@ -12052,8 +12045,8 @@ def err_unsupported_coopmat_binary_operator
def err_unsupported_coopmat_scalar_operator
: Error<"unsupported cooperative matrix scalar operator">;
-def err_ref_coopmat_var
- : Error<"invalid reference of cooperative matrix variable">;
+def err_invalid_operand_for_coopmat_scalar_operator
+ : Error<"unsupported type for cooperative matrix scalar operator">;
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 2b12d11e9bb323..bf04230f88d45a 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3113,17 +3113,17 @@ class Sema final : public SemaBase {
bool BuiltinCountedByRef(CallExpr *TheCall);
// Coop matrix handling.
- void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ bool CheckCoopMatrixLoadElementType(QualType MatrixType,
SourceLocation MatrixLoc, CallExpr *call);
- void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ bool CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
QualType BufferType,
SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- bool CheckCoopMatrixLoadStoreStride(Expr *Stride);
+ bool CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx);
ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
- void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ bool CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
@@ -8049,7 +8049,7 @@ class Sema final : public SemaBase {
/// Are the two types cooperative matrix types and do they have the same
/// dimensions i.e. do they have the same number of rows and the same number
/// of columns? Also do they have the same scope and use?
- bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ bool areCoopMatrixTypesCompatible(QualType srcTy, QualType destTy);
bool areVectorTypesSameSize(QualType srcType, QualType destType);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 4b39ef832298d0..4b2a024377c68d 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17473,35 +17473,38 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
}
// Check coop_mat_load/store matrix element has same type with buffer pointer.
-void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
QualType BufferType,
SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
- return;
+ return true;
}
assert(isa<PointerType>(BufferType));
auto *PTy = BufferType->castAs<PointerType>();
if (MTy->getElementType().getUnqualifiedType() !=
- PTy->getPointeeType().getUnqualifiedType())
+ PTy->getPointeeType().getUnqualifiedType()) {
Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+ return true;
+ }
+ return false;
}
-void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
SourceLocation MatrixLoc,
- CallExpr *call) {
-
- FunctionDecl *F = call->getDirectCallee();
+ CallExpr *Call) {
+ FunctionDecl *F = Call->getDirectCallee();
assert(F);
DeclarationName MemberName = F->getDeclName();
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (Fname->isStr("coop_mat_load"))
- CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ return CheckCoopMatrixLoadStoreElementType(MatrixType, Call->getArg(0)->getType(),
+ MatrixLoc);
+ return false;
}
// Check coop_mat_load/store layout argument
@@ -17524,11 +17527,14 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
-bool Sema::CheckCoopMatrixLoadStoreStride(Expr *Stride) {
+bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
+ assert(TheCall->getNumArgs() >= ArgIdx + 1);
+ Expr *Stride = TheCall->getArg(ArgIdx);
if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
return true;
}
+ TheCall->setArg(ArgIdx, Stride);
return false;
}
@@ -17540,7 +17546,7 @@ ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
return ExprError();
- if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(2)))
+ if (CheckCoopMatrixLoadStoreStride(TheCall, 2 /*ArgIdx*/))
return ExprError();
return CallResult;
}
@@ -17553,23 +17559,24 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
- CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc()))
+ return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
- if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(3)))
+ if (CheckCoopMatrixLoadStoreStride(TheCall, 3 /*ArgIdx*/))
return ExprError();
return CallResult;
}
-void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
FunctionDecl *F = TheCall->getDirectCallee();
assert(F);
DeclarationName MemberName = F->getDeclName();
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (!Fname->isStr("coop_mat_mulAdd"))
- return;
+ return false;
auto MC = TheCall->getArg(2);
auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
@@ -17581,17 +17588,24 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
if (!M2Ty)
Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
if (!MOutTy || !M2Ty)
- return;
+ return true;
- if (MOutTy->getUse() != 2)
+ if (MOutTy->getUse() != 2) {
Diag(Loc, diag::err_coop_matrix_useACC);
+ return true;
+ }
if (MOutTy->getElementType().getUnqualifiedType() !=
- M2Ty->getElementType().getUnqualifiedType())
+ M2Ty->getElementType().getUnqualifiedType()) {
Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
+ return true;
+ }
- if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesCompatible(TheCall->getType(), MC->getType())) {
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+ return false;
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
@@ -17605,7 +17619,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesCompatible(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17660,6 +17674,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!OperandsTy->isEnumeralType()) {
Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ return ExprError();
}
}
@@ -17682,12 +17697,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!M0Ty || !M1Ty || !M2Ty)
return ExprError();
- if (M0Ty->getUse() != 0)
+ if (M0Ty->getUse() != 0) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
- if (M1Ty->getUse() != 1)
+ return ExprError();
+ }
+ if (M1Ty->getUse() != 1) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
- if (M2Ty->getUse() != 2)
+ return ExprError();
+ }
+ if (M2Ty->getUse() != 2) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+ return ExprError();
+ }
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType())
@@ -17712,6 +17733,25 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+
+ if (!M0Ty) {
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ return ExprError();
+ }
+
+ Expr *Arg1 = TheCall->getArg(1);
+ QualType Ty = Arg1->getType();
+ if (!Ty->isScalarType()) {
+ Diag(Arg1->getBeginLoc(),
+ diag::err_invalid_operand_for_coopmat_scalar_operator);
+ return ExprError();
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ Ty.getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
TheCall->setType(Arg0->getType());
return CallResult;
@@ -17723,6 +17763,11 @@ ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ if (!M0Ty) {
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ return ExprError();
+ }
TheCall->setType(Arg0->getType());
return CallResult;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 12670978778aff..3896b6f3019c67 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14105,18 +14105,14 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if cooperative matrix builtin call returns cooperative matrix.
+// Return true if builtin call returns cooperative matrix.
bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
- auto call = dyn_cast<CallExpr>(RHSExpr);
- if (!call)
+ auto Call = dyn_cast<CallExpr>(RHSExpr);
+ if (!Call)
return false;
- FunctionDecl *F = call->getDirectCallee();
+ FunctionDecl *F = Call->getDirectCallee();
if (!F)
return false;
- DeclarationName MemberName = F->getDeclName();
- IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
- if (!Fname)
- return false;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
case Builtin::BIcoop_mat_mulAdd:
@@ -14131,8 +14127,6 @@ bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
default:
return false;
}
-
- return false;
}
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 6ac28d166042bc..553a218beaa4fb 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -8068,8 +8068,7 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
-bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
- QualType destTy) {
+bool Sema::areCoopMatrixTypesCompatible(QualType srcTy, QualType destTy) {
if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
return false;
@@ -8192,7 +8191,7 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind) {
if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
- if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ if (!areCoopMatrixTypesCompatible(SrcTy, DestTy)) {
return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
<< DestTy << SrcTy << R;
}
@@ -16003,12 +16002,14 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
- auto call = dyn_cast<CallExpr>(RHSExpr);
- assert(call);
- call->setType(LHSExpr->getType());
- CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
- CheckCoopMatrixMatMulOutput(call);
+ auto Call = dyn_cast<CallExpr>(RHSExpr);
+ assert(Call);
+ Call->setType(LHSExpr->getType());
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ Call))
+ return ExprError();
+ if (CheckCoopMatrixMatMulOutput(Call))
+ return ExprError();
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
>From 9c49a6313502d938a3e66e89a742fa7fe02767cb Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 20:42:05 -0400
Subject: [PATCH 31/38] Fix clang format issues
---
clang/lib/Sema/SemaChecking.cpp | 7 +++----
clang/lib/Sema/SemaExpr.cpp | 4 ++--
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 4b2a024377c68d..c82014e0c820b6 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17502,8 +17502,8 @@ bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (Fname->isStr("coop_mat_load"))
- return CheckCoopMatrixLoadStoreElementType(MatrixType, Call->getArg(0)->getType(),
- MatrixLoc);
+ return CheckCoopMatrixLoadStoreElementType(
+ MatrixType, Call->getArg(0)->getType(), MatrixLoc);
return false;
}
@@ -17749,8 +17749,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
}
- if (M0Ty->getElementType().getUnqualifiedType() !=
- Ty.getUnqualifiedType())
+ if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
TheCall->setType(Arg0->getType());
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 553a218beaa4fb..32213a64ea708b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16005,8 +16005,8 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
auto Call = dyn_cast<CallExpr>(RHSExpr);
assert(Call);
Call->setType(LHSExpr->getType());
- if (CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- Call))
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
+ LHSExpr->getBeginLoc(), Call))
return ExprError();
if (CheckCoopMatrixMatMulOutput(Call))
return ExprError();
>From 69a813d6a9bf2e931637aa138add04ced9053027 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 14:57:35 -0400
Subject: [PATCH 32/38] Fix issue caused by recent merge to the tip.
---
clang/lib/Sema/SemaChecking.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index c82014e0c820b6..8f00e04385b78f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17530,7 +17530,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
- if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
+ if (convertArgumentToType(Stride, Context.getSizeType())) {
Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
return true;
}
>From 8128fbf3fda7d81269b2fb728dac174bb592774b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 16:35:51 -0400
Subject: [PATCH 33/38] Add missing check in TypePrinter.cpp
---
clang/lib/AST/TypePrinter.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 34380e3c8965f8..047ebdc71d7d98 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2178,6 +2178,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
case attr::ExtVectorType:
OS << "ext_vector_type";
break;
+ case attr::CoopMatrixType:
+ OS << "coop_mat";
+ break;
case attr::CFISalt:
OS << "cfi_salt(\"" << cast<CFISaltAttr>(T->getAttr())->getSalt() << "\")";
break;
>From 0d172a642586deb373aa2aaba9e15945af7048a8 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 18:07:29 -0400
Subject: [PATCH 34/38] Update test
---
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index f77fe542fd210f..10fd9c08173b58 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -31,10 +31,6 @@
// expected-no-diagnostics
void test_enum_constants(void) {
- // coop_matrix_scope_t
- coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
- (void)s;
-
// coop_matrix_use_t
coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
@@ -53,7 +49,7 @@ void test_enum_constants(void) {
// ── 2d. Enum constant values match the spec ──────────────────────────────────
void test_enum_values(void) {
- _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(memory_scope_sub_group == 4, "scope subgroup");
_Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
_Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
_Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
>From 2f0e229c6d8eade085f223d91c3361e64adef0e7 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 29 Sep 2026 12:37:47 -0400
Subject: [PATCH 35/38] Fix minor build issue
---
clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index d89c80ca01754f..7a0b81494cf1e0 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -488,6 +488,7 @@ void ExprEngine::VisitCastExpr(const CastExpr *CastE, ExplodedNode *Pred,
// Various C++ casts that are not handled yet.
case CK_ToUnion:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_VectorSplat:
case CK_HLSLElementwiseCast:
case CK_HLSLAggregateSplatCast:
>From 9a8fdf33d6af3864c72c0cddb1970cf3545c1cfc Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:06:37 -0400
Subject: [PATCH 36/38] Fix minor merge issue
---
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index f7202caf8e5f30..f7fbd0a42e7a26 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -2341,9 +2341,6 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
case Builtin::BIcoop_mat_scalar_neg:
case Builtin::BIcoop_mat_init:
case Builtin::BIcoop_mat_length:
- case Builtin::BI__builtin_matrix_transpose:
- case Builtin::BI__builtin_matrix_column_major_load:
- case Builtin::BI__builtin_matrix_column_major_store:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
>From e3f0ee4d8895c609364eef2631efba269cc5ae19 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:07:46 -0400
Subject: [PATCH 37/38] Fix issue of direct initialization of cooperative
matrix
---
clang/lib/Sema/SemaDecl.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 3896b6f3019c67..8c6e7d709d0e0f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14345,6 +14345,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
Init = Result.get();
}
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
+
// Perform the initialization.
bool InitializedFromParenListExpr = false;
bool IsParenListInit = false;
@@ -14647,19 +14660,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
-
- // Set return type of builtin call using type of LHS variable.
- // This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
- if (!VDecl->getType()->isCooperativeMatrixType()) {
- Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
- return;
- }
-
- auto call = dyn_cast<CallExpr>(Init);
- assert(call);
- call->setType(VDecl->getType());
- }
}
void Sema::ActOnInitializerError(Decl *D) {
>From 2c9dc2cd333a58a168991532820030106198307b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:43:36 -0400
Subject: [PATCH 38/38] Update correct emission of MatrixOperand operand for
coop_mat_mulAdd
---
clang/include/clang/Basic/AttrDocs.td | 2 +-
clang/lib/CodeGen/CGBuiltin.cpp | 16 ++++++----------
2 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 9023b2f594ebb3..b9eb5606319a82 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1521,7 +1521,7 @@ chmat44 vmat2;
The base component type must be a scalar numerical type.
Implicit and explicit casts between cooperative matrix types are not generally permitted.
Conversion between cooperative matrix types is allowed only when the types have identical scope, dimensions, and use parameters, and their component types are compatible according to the OpenCL C conversion rules.
-Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_matmul_add, and coop_mat_length.
+Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_mat_mulAdd, coop_mat_length and element-wise arithmetic operations.
See the OpenCL documentation for some more complete examples.
}];
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index c3c18e616218b7..47c286f16f9b1b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -4795,27 +4795,23 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
auto Arg2 = E->getArg(2);
+ Value *Zero = Constant::getNullValue(Int32Ty);
auto MA = EmitScalarExpr(Arg0);
auto MB = EmitScalarExpr(Arg1);
auto MC = EmitScalarExpr(Arg2);
+ auto MatrixOperand =
+ (E->getNumArgs() == 4) ? EmitScalarExpr(E->getArg(3)) : Zero;
+
const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
- // Check if the data is signed/unsigned
- QualType QT = MATy->getElementType();
- bool isSigned = false;
- if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
- isSigned = BT->isSignedInteger();
- }
- llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
- llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
auto *RetType = getTargetExtType(CGM, MTy);
auto *AType = getTargetExtType(CGM, MATy);
auto *BType = getTargetExtType(CGM, MBTy);
auto *CType = getTargetExtType(CGM, MCTy);
llvm::FunctionType *FTy =
- llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int32Ty}, false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
@@ -4826,7 +4822,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
- auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, MatrixOperand});
NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
return RValue::get(NewCall);
}
More information about the cfe-commits
mailing list