[clang] [lldb] [llvm] [OpenCL][Clang] Add support for cooperative matrix extension (PR #221328)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Sep 8 14:01:19 PDT 2026
https://github.com/asudarsa-qti updated https://github.com/llvm/llvm-project/pull/221328
>From 4b083de73e423c71089068e74c99c3d9f10fcf9f Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:14:42 -0400
Subject: [PATCH 01/17] [OpenCL][CoopMat] Patch 1 - Introduce Cooperative
matrix type
---
clang/include/clang/AST/ASTContext.h | 10 +++
clang/include/clang/AST/RecursiveASTVisitor.h | 11 +++
clang/include/clang/AST/TypeBase.h | 90 +++++++++++++++++++
clang/include/clang/AST/TypeLoc.h | 34 ++++++-
clang/include/clang/AST/TypeProperties.td | 13 +++
clang/include/clang/Basic/TypeNodes.td | 1 +
.../clang/Serialization/TypeBitCodes.def | 1 +
clang/lib/AST/ASTContext.cpp | 82 +++++++++++++++++
clang/lib/AST/Type.cpp | 38 ++++++++
clang/lib/AST/TypePrinter.cpp | 16 ++++
10 files changed, 295 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index f875ae365b892..985bf3af2b14d 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -246,6 +246,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
mutable llvm::ContextualFoldingSet<DependentVectorType, ASTContext &>
DependentVectorTypes;
mutable llvm::FoldingSet<ConstantMatrixType> MatrixTypes;
+ mutable llvm::FoldingSet<CooperativeMatrixType> CooperativeMatrixTypes;
mutable llvm::ContextualFoldingSet<DependentSizedMatrixType, ASTContext &>
DependentSizedMatrixTypes;
mutable llvm::FoldingSet<FunctionNoProtoType> FunctionNoProtoTypes;
@@ -1875,6 +1876,15 @@ class ASTContext : public RefCountedBase<ASTContext> {
QualType getConstantMatrixType(QualType ElementType, unsigned NumRows,
unsigned NumColumns) const;
+ /// Return the unique reference to the cooperative matrix type of the
+ /// specified element type and size.
+ ///
+ /// \pre \p ElementType must be a valid matrix element type (see
+ /// MatrixType::isValidElementType).
+ QualType getCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use) const;
+
/// Return the unique reference to the matrix type of the specified element
/// type and size
QualType getDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index acb50d97fed09..892583d7530a9 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -1089,6 +1089,9 @@ DEF_TRAVERSE_TYPE(ExtVectorType, { TRY_TO(TraverseType(T->getElementType())); })
DEF_TRAVERSE_TYPE(ConstantMatrixType,
{ TRY_TO(TraverseType(T->getElementType())); })
+DEF_TRAVERSE_TYPE(CooperativeMatrixType,
+ { TRY_TO(TraverseType(T->getElementType())); })
+
DEF_TRAVERSE_TYPE(DependentSizedMatrixType, {
if (T->getRowExpr())
TRY_TO(TraverseStmt(T->getRowExpr()));
@@ -1409,6 +1412,14 @@ DEF_TRAVERSE_TYPELOC(ConstantMatrixType, {
TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
})
+DEF_TRAVERSE_TYPELOC(CooperativeMatrixType, {
+ TRY_TO(TraverseStmt(TL.getAttrScopeOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrUseOperand()));
+ TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
+})
+
DEF_TRAVERSE_TYPELOC(DependentSizedMatrixType, {
TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index e69173a1fdd6c..4b3fba1f9934c 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -2708,6 +2708,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
bool isSubscriptableVectorType() const;
bool isMatrixType() const; // Matrix type.
bool isConstantMatrixType() const; // Constant matrix type.
+ bool isCooperativeMatrixType() const; // Cooperative matrix type.
bool isOverflowBehaviorType() const; // Overflow behavior type.
bool isDependentAddressSpaceType() const; // value-dependent address space qualifier
bool isObjCObjectPointerType() const; // pointer to ObjC object
@@ -4493,6 +4494,7 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
+ T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4582,6 +4584,90 @@ class ConstantMatrixType final : public MatrixType {
}
};
+/// Represents a cooperative matrix type.
+class CooperativeMatrixType final : public MatrixType {
+protected:
+ friend class ASTContext;
+
+ /// Number of rows and columns.
+ unsigned NumRows;
+ unsigned NumColumns;
+
+ /// Scope and use
+ unsigned Scope;
+ unsigned Use;
+
+ static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
+
+ CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
+ unsigned NRows, unsigned NColumns, unsigned Use,
+ QualType CanonElementType);
+
+ CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
+ unsigned Scope, unsigned NRows, unsigned NColumns,
+ unsigned Use, QualType CanonElementType);
+
+public:
+ /// Returns the number of rows in the matrix.
+ unsigned getNumRows() const { return NumRows; }
+
+ /// Returns the number of columns in the matrix.
+ unsigned getNumColumns() const { return NumColumns; }
+
+ /// Returns the scope of the matrix.
+ unsigned getScope() const { return Scope; }
+
+ /// Returns the use of the matrix.
+ unsigned getUse() const { return Use; }
+
+ /// Returns the number of elements required to embed the matrix into a vector.
+ unsigned getNumElementsFlattened() const {
+ return getNumRows() * getNumColumns();
+ }
+
+ /// Returns true if \p NumElements is a valid matrix dimension.
+ static constexpr bool isDimensionValid(size_t NumElements) {
+ return NumElements > 0 && NumElements <= MaxElementsPerDimension;
+ }
+
+ /// Return true if \p Scope is valid
+ static constexpr bool isScopeValid(size_t Scope) {
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ }
+
+ /// Return true if \p Use is valid
+ static constexpr bool isUseValid(size_t Use) {
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ }
+
+ /// Returns the maximum number of elements per dimension.
+ static constexpr unsigned getMaxElementsPerDimension() {
+ return MaxElementsPerDimension;
+ }
+
+ void Profile(llvm::FoldingSetNodeID &ID) {
+ Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ getUse(), getTypeClass());
+ }
+
+ static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
+ unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Use, TypeClass TypeClass) {
+ ID.AddPointer(ElementType.getAsOpaquePtr());
+ ID.AddInteger(NumRows);
+ ID.AddInteger(NumColumns);
+ ID.AddInteger(Scope);
+ ID.AddInteger(Use);
+ ID.AddInteger(TypeClass);
+ }
+
+ static bool classof(const Type *T) {
+ return T->getTypeClass() == CooperativeMatrix;
+ }
+};
+
/// Represents a matrix type where the type and the number of rows and columns
/// is dependent on a template.
class DependentSizedMatrixType final : public MatrixType {
@@ -8907,6 +8993,10 @@ inline bool Type::isConstantMatrixType() const {
return isa<ConstantMatrixType>(CanonicalType);
}
+inline bool Type::isCooperativeMatrixType() const {
+ return isa<CooperativeMatrixType>(CanonicalType);
+}
+
inline bool Type::isOverflowBehaviorType() const {
return isa<OverflowBehaviorType>(CanonicalType);
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index b3cb4d0f33629..24fcda8f4c7e5 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,8 +2142,10 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
+ Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
+ Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2151,26 +2153,50 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^~~~~~~~~~~~~~~~~
+ /// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2187,8 +2213,10 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
}
};
@@ -2196,6 +2224,10 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
+class CooperativeMatrixTypeLoc
+ : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType> {};
+
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index dc2a45ec85729..29ea67a21b299 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -260,6 +260,19 @@ let Class = ConstantMatrixType in {
}]>;
}
+let Class = CooperativeMatrixType in {
+ def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
+ def : Property<"numColumns", UInt32> {
+ let Read = [{ node->getNumColumns() }];
+ }
+ def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+
+ def : Creator<[{
+ return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ }]>;
+}
+
let Class = DependentSizedMatrixType in {
def : Property<"rows", ExprRef> {
let Read = [{ node->getRowExpr() }];
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 700a73f669690..e6f396245070c 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,6 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<MatrixType>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index 9f1da65e0f940..c9ebac63390f5 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -70,5 +70,6 @@ TYPE_BIT_CODE(HLSLInlineSpirv, HLSL_INLINE_SPIRV, 60)
TYPE_BIT_CODE(PredefinedSugar, PREDEFINED_SUGAR, 61)
TYPE_BIT_CODE(SubstBuiltinTemplatePack, SUBST_BUILTIN_TEMPLATE_PACK, 62)
TYPE_BIT_CODE(OverflowBehavior, OVERFLOWBEHAVIOR, 63)
+TYPE_BIT_CODE(CooperativeMatrix, COOPERATIVE_MATRIX, 64)
#undef TYPE_BIT_CODE
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index b7e771595e86e..c4839bb07c89b 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2202,6 +2202,17 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
break;
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(T);
+ TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
+ // The internal layout of a matrix value is implementation defined.
+ // Initially be ABI compatible with arrays with respect to alignment and
+ // size.
+ Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
+ Align = ElementInfo.Align;
+ break;
+ }
+
case Type::Builtin:
switch (cast<BuiltinType>(T)->getKind()) {
default: llvm_unreachable("Unknown builtin type!");
@@ -3522,6 +3533,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
case Type::Pipe:
case Type::BitInt:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
OS << "?";
return;
@@ -4344,6 +4356,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
case Type::ExtVector:
case Type::DependentSizedExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::ObjCObject:
@@ -4890,6 +4903,45 @@ QualType ASTContext::getConstantMatrixType(QualType ElementTy, unsigned NumRows,
return QualType(New, 0);
}
+QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
+ unsigned Scope, unsigned NumRows,
+ unsigned NumColumns,
+ unsigned Use) const {
+ llvm::FoldingSetNodeID ID;
+ CooperativeMatrixType::Profile(ID, ElementTy, Scope, NumRows, NumColumns, Use,
+ Type::CooperativeMatrix);
+
+ assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
+ "need a valid element type");
+ assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
+ CooperativeMatrixType::isDimensionValid(NumColumns) &&
+ "need valid matrix dimensions");
+ assert(CooperativeMatrixType::isScopeValid(Scope) &&
+ "need valid matrix scope");
+ assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
+ void *InsertPos = nullptr;
+ if (CooperativeMatrixType *MTP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ return QualType(MTP, 0);
+
+ QualType Canonical;
+ if (!ElementTy.isCanonical()) {
+ Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
+ NumRows, NumColumns, Use);
+
+ CooperativeMatrixType *NewIP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ assert(!NewIP && "Matrix type shouldn't already exist in the map");
+ (void)NewIP;
+ }
+
+ auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
+ ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
+ CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ Types.push_back(New);
+ return QualType(New, 0);
+}
+
QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
Expr *RowExpr,
Expr *ColumnExpr,
@@ -9759,6 +9811,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
return;
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
if (NotEncodedT)
*NotEncodedT = T;
return;
@@ -10727,6 +10780,18 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
+/// areCompatMatrixTypes - Return true if the two specified matrix types are
+/// compatible.
+static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
+ assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
+ return LHS->getElementType() == RHS->getElementType() &&
+ LHS->getScope() == RHS->getScope() &&
+ LHS->getNumRows() == RHS->getNumRows() &&
+ LHS->getNumColumns() == RHS->getNumColumns() &&
+ LHS->getUse() == RHS->getUse();
+}
+
bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
QualType SecondVec) {
assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
@@ -12241,6 +12306,11 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
RHSCan->castAs<ConstantMatrixType>()))
return LHS;
return {};
+ case Type::CooperativeMatrix:
+ if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
+ return LHS;
+ return {};
case Type::ObjCObject: {
// Check if the types are assignment compatible.
// FIXME: This should be type compatibility, e.g. whether
@@ -14605,6 +14675,17 @@ static QualType getCommonNonSugarTypeNode(const ASTContext &Ctx, const Type *X,
return Ctx.getConstantMatrixType(getCommonElementType(Ctx, MX, MY),
MX->getNumRows(), MX->getNumColumns());
}
+ case Type::CooperativeMatrix: {
+ const auto *MX = cast<CooperativeMatrixType>(X),
+ *MY = cast<CooperativeMatrixType>(Y);
+ assert(MX->getScope() == MY->getScope());
+ assert(MX->getNumRows() == MY->getNumRows());
+ assert(MX->getNumColumns() == MY->getNumColumns());
+ assert(MX->getUse() == MY->getUse());
+ return Ctx.getCooperativeMatrixType(getCommonElementType(Ctx, MX, MY),
+ MX->getScope(), MX->getNumRows(),
+ MX->getNumColumns(), MX->getUse());
+ }
case Type::DependentSizedMatrix: {
const auto *MX = cast<DependentSizedMatrixType>(X),
*MY = cast<DependentSizedMatrixType>(Y);
@@ -14759,6 +14840,7 @@ static QualType getCommonSugarTypeNode(const ASTContext &Ctx, const Type *X,
CANONICAL_TYPE(ConstantArray)
CANONICAL_TYPE(ArrayParameter)
CANONICAL_TYPE(ConstantMatrix)
+ CANONICAL_TYPE(CooperativeMatrix)
CANONICAL_TYPE(Enum)
CANONICAL_TYPE(ExtVector)
CANONICAL_TYPE(FunctionNoProto)
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index df296eb4e28e5..4e50e1a57e42c 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -423,6 +423,20 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns) {}
+CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
+ nColumns, use, canonType) {}
+
+CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : MatrixType(tc, matrixType, canonType), NumRows(nRows),
+ NumColumns(nColumns), Scope(scope), Use(use) {}
+
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -1190,6 +1204,18 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
T->getNumColumns());
}
+ QualType VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ QualType elementType = recurse(T->getElementType());
+ if (elementType.isNull())
+ return {};
+ if (elementType.getAsOpaquePtr() == T->getElementType().getAsOpaquePtr())
+ return QualType(T, 0);
+
+ return Ctx.getCooperativeMatrixType(elementType, T->getScope(),
+ T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ }
+
QualType VisitOverflowBehaviorType(const OverflowBehaviorType *T) {
QualType UnderlyingType = recurse(T->getUnderlyingType());
if (UnderlyingType.isNull())
@@ -2097,6 +2123,10 @@ class GetContainedDeducedTypeVisitor
return Visit(T->getElementType());
}
+ Type *VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+ }
+
Type *VisitFunctionProtoType(const FunctionProtoType *T) {
if (Syntactic && T->hasTrailingReturn())
return const_cast<FunctionProtoType *>(T);
@@ -3178,6 +3208,8 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
+ if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
+ return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
@@ -5024,6 +5056,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
return Cache::get(cast<VectorType>(T)->getElementType());
case Type::ConstantMatrix:
return Cache::get(cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return Cache::get(cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return Cache::get(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5126,6 +5160,9 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
case Type::ConstantMatrix:
return computeTypeLinkageInfo(
cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return computeTypeLinkageInfo(
+ cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return computeTypeLinkageInfo(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5327,6 +5364,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::FunctionProto:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 80d63434c28b1..df65c340a89e4 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -279,6 +279,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -917,6 +918,20 @@ void TypePrinter::printConstantMatrixAfter(const ConstantMatrixType *T,
printAfter(T->getElementType(), OS);
}
+void TypePrinter::printCooperativeMatrixBefore(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printBefore(T->getElementType(), OS);
+ OS << " __attribute__((coop_mat(";
+ OS << T->getScope() << ", " << T->getNumRows() << ", ";
+ OS << T->getNumColumns() << ", " << T->getUse();
+ OS << ")))";
+}
+
+void TypePrinter::printCooperativeMatrixAfter(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printAfter(T->getElementType(), OS);
+}
+
void TypePrinter::printDependentSizedMatrixBefore(
const DependentSizedMatrixType *T, raw_ostream &OS) {
if (Policy.UseHLSLTypes) {
@@ -2043,6 +2058,7 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
case attr::OpenCLConstantAddressSpace:
case attr::OpenCLGenericAddressSpace:
case attr::HLSLGroupSharedAddressSpace:
+ case attr::CoopMatrixType:
// FIXME: Update printAttributedBefore to print these once we generate
// AttributedType nodes for them.
break;
>From 1141ef766700970358084e9aa40ea5a9b8803dc2 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:15:43 -0400
Subject: [PATCH 02/17] Patch 2 - Define extension macro
---
.../include/clang/Basic/OpenCLExtensions.def | 1 +
clang/lib/Headers/opencl-c-base.h | 23 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index f8af9b843f95f..89e96dca85a59 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,6 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
+OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 251894ac5a263..7b3d239299c6e 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -771,4 +771,27 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From fac3d1709d41e14c83bba83c5f159d8fb5bd1322 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:16:39 -0400
Subject: [PATCH 03/17] Patch 3 - Add support for serialization, round-trip and
mangling
---
clang/lib/AST/ASTImporter.cpp | 11 +++++++++
clang/lib/AST/ASTStructuralEquivalence.cpp | 15 +++++++++++++
clang/lib/AST/ExprConstant.cpp | 1 +
clang/lib/AST/ItaniumMangle.cpp | 26 ++++++++++++++++++++++
clang/lib/AST/MicrosoftMangle.cpp | 21 +++++++++++++++++
clang/lib/Serialization/ASTReader.cpp | 10 +++++++++
clang/lib/Serialization/ASTWriter.cpp | 12 ++++++++++
7 files changed, 96 insertions(+)
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 5778b0996c36a..0d2f171954a91 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -2106,6 +2106,17 @@ ExpectedType clang::ASTNodeImporter::VisitConstantMatrixType(
*ToElementTypeOrErr, T->getNumRows(), T->getNumColumns());
}
+ExpectedType clang::ASTNodeImporter::VisitCooperativeMatrixType(
+ const clang::CooperativeMatrixType *T) {
+ ExpectedType ToElementTypeOrErr = import(T->getElementType());
+ if (!ToElementTypeOrErr)
+ return ToElementTypeOrErr.takeError();
+
+ return Importer.getToContext().getCooperativeMatrixType(
+ *ToElementTypeOrErr, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+}
+
ExpectedType clang::ASTNodeImporter::VisitDependentAddressSpaceType(
const clang::DependentAddressSpaceType *T) {
Error Err = Error::success();
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 029e33ae0db76..8ec1100f9f4d8 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1151,6 +1151,21 @@ bool ASTStructuralEquivalence::isEquivalent(
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
+ const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
+ // The element types must be structurally equivalent and the number of rows
+ // and columns must match.
+ if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
+ Mat2->getElementType()) ||
+ Mat1->getScope() != Mat2->getScope() ||
+ Mat1->getNumRows() != Mat2->getNumRows() ||
+ Mat1->getNumColumns() != Mat2->getNumColumns() ||
+ Mat1->getUse() != Mat2->getUse())
+ return false;
+ break;
+ }
+
case Type::FunctionProto: {
const auto *Proto1 = cast<FunctionProtoType>(T1);
const auto *Proto2 = cast<FunctionProtoType>(T2);
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 43d49da015d4e..3d700e25afe66 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -16423,6 +16423,7 @@ GCCTypeClass EvaluateBuiltinClassifyType(QualType T,
case Type::BlockPointer:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::ObjCObject:
case Type::ObjCInterface:
case Type::ObjCObjectPointer:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 3a3cde3448f44..3869c4d857d65 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2483,6 +2483,7 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -4441,6 +4442,31 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
Out << "E";
}
+void CXXNameMangler::mangleType(const CooperativeMatrixType *T) {
+ // Mangle cooperative matrix types as a vendor extended type:
+ // u<Len>coop_matI<Scope><Rows><Columns><Use><element type>E
+
+ mangleVendorType("coop_mat");
+
+ Out << "I";
+ auto &ASTCtx = getASTContext();
+ unsigned BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
+ llvm::APSInt Scope(BitWidth);
+ Scope = T->getScope();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Scope);
+ llvm::APSInt Rows(BitWidth);
+ Rows = T->getNumRows();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Rows);
+ llvm::APSInt Columns(BitWidth);
+ Columns = T->getNumColumns();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Columns);
+ llvm::APSInt Use(BitWidth);
+ Use = T->getUse();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Use);
+ mangleType(T->getElementType());
+ Out << "E";
+}
+
void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
// Mangle matrix types as a vendor extended type:
// u<Len>matrix_typeI<row expr><column expr><element type>E
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index cc7bf2279b72e..2a41f601b2802 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3775,6 +3775,27 @@ void MicrosoftCXXNameMangler::mangleType(const ConstantMatrixType *T,
mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
}
+void MicrosoftCXXNameMangler::mangleType(const CooperativeMatrixType *T,
+ Qualifiers quals, SourceRange Range) {
+ QualType EltTy = T->getElementType();
+
+ llvm::SmallString<64> TemplateMangling;
+ llvm::raw_svector_ostream Stream(TemplateMangling);
+ MicrosoftCXXNameMangler Extra(Context, Stream);
+
+ Stream << "?$";
+
+ Extra.mangleSourceName("__coop_mat");
+ Extra.mangleType(EltTy, Range, QMM_Escape);
+
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getScope()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumRows()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumColumns()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getUse()));
+
+ mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
+}
+
void MicrosoftCXXNameMangler::mangleType(const DependentSizedMatrixType *T,
Qualifiers quals, SourceRange Range) {
Error(Range.getBegin(), "dependent-sized matrix type") << Range;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a9c230d767c50..bbb31f8f26aac 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7593,6 +7593,16 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ TL.setAttrNameLoc(readSourceLocation());
+ TL.setAttrOperandParensRange(readSourceRange());
+ TL.setAttrScopeOperand(Reader.readExpr());
+ TL.setAttrRowOperand(Reader.readExpr());
+ TL.setAttrColumnOperand(Reader.readExpr());
+ TL.setAttrUseOperand(Reader.readExpr());
+}
+
void TypeLocReader::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index de985b770cb01..0f14f50a1ece8 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -454,6 +454,18 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ addSourceLocation(TL.getAttrNameLoc());
+ SourceRange range = TL.getAttrOperandParensRange();
+ addSourceLocation(range.getBegin());
+ addSourceLocation(range.getEnd());
+ Record.AddStmt(TL.getAttrScopeOperand());
+ Record.AddStmt(TL.getAttrRowOperand());
+ Record.AddStmt(TL.getAttrColumnOperand());
+ Record.AddStmt(TL.getAttrUseOperand());
+}
+
void TypeLocWriter::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
>From 64aebdb31791d13be0f255c2ab765c4b295d8c46 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:21 -0400
Subject: [PATCH 04/17] Patch 4 - Add support for semantic analysis and
checking
---
clang/include/clang/Basic/Attr.td | 8 +
.../clang/Basic/DiagnosticSemaKinds.td | 54 +++
clang/include/clang/Sema/Sema.h | 34 +-
clang/lib/Sema/SemaChecking.cpp | 310 +++++++++++++++++-
clang/lib/Sema/SemaDecl.cpp | 32 ++
clang/lib/Sema/SemaExpr.cpp | 131 +++++++-
clang/lib/Sema/SemaLookup.cpp | 1 +
clang/lib/Sema/SemaTemplate.cpp | 5 +
clang/lib/Sema/SemaTemplateDeduction.cpp | 31 ++
clang/lib/Sema/SemaType.cpp | 78 ++++-
clang/lib/Sema/TreeTransform.h | 42 +++
11 files changed, 704 insertions(+), 22 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 66142ed95090a..342c73064a998 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3898,6 +3898,14 @@ def MatrixType : TypeAttr {
let PragmaAttributeSupport = 0;
}
+def CoopMatrixType : TypeAttr {
+ let Spellings = [Clang<"coop_mat">];
+ let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
+ let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
+ UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
+ let Documentation = [Undocumented];
+}
+
def Visibility : InheritableAttr {
let Clone = 0;
let Spellings = [GCC<"visibility">];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6c4339b6175eb..5de45ae4929ec 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11947,6 +11947,60 @@ def ext_opencl_ext_vector_type_rgba_selector: ExtWarn<
def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
+// OpenCL C Cooperative Matrix.
+def err_invalid_coopmat_attr
+ : Error<"invalid argument of cooperative matrix attribute">;
+
+def err_coop_mat_not_primitive
+ : Error<"cooperative matrix attribute only supported for integer and "
+ "floating-point types">;
+
+def err_coop_mat_rol_col_size
+ : Error<"cooperative matrix row and column must have power-of-two size">;
+
+def err_coop_mem_layout_enum
+ : Error<"memory layout for cooperative matrix should be "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+
+def err_coop_element_and_pointer_type
+ : Error<"inconsistent between cooperative matrix element type and buffer "
+ "pointer type">;
+
+def err_coop_matrix_arg
+ : Error<"argument must be a valid cooperative matrix type">;
+
+def err_coop_matrix_assignment
+ : Error<"builtin return value should be assigned to cooperative matrix "
+ "type variable">;
+
+def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_A">;
+
+def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_B">;
+
+def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+
+def err_coop_matrix_element_type
+ : Error<"inconsistent cooperative matrix element type">;
+
+def err_coop_matrix_row_or_col_mismatch
+ : Error<"mismatch of cooperative matrix row or column">;
+
+def err_coop_matrix_use_type
+ : Error<"inconsistent cooperative matrix use type">;
+
+def err_unsupported_coopmat_binary_operator
+ : Error<"unsupported cooperative matrix binary operator">;
+
+def err_unsupported_coopmat_scalar_operator
+ : Error<"unsupported cooperative matrix scalar operator">;
+
+def err_ref_coopmat_var
+ : Error<"invalid reference of cooperative matrix variable">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 9b07b591b8c07..b10a5b3f914c1 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3060,6 +3060,34 @@ class Sema final : public SemaBase {
BuiltinCountedByRefKind K);
bool BuiltinCountedByRef(CallExpr *TheCall);
+ // Coop matrix handling.
+ void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc, CallExpr *call);
+ void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc);
+ bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
+ bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult);
+ void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc);
+ ExprResult BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatScalarOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ ExprResult BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+
// Matrix builtin handling.
ExprResult BuiltinMatrixTranspose(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
@@ -15288,8 +15316,10 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
+
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc);
+ SourceLocation AttrLoc, Expr *Scope = nullptr,
+ Expr *Use = nullptr, bool IsCoopMat = false);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
@@ -15312,6 +15342,8 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
+ bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+
/// Build a function type.
///
/// This routine checks the function type according to C++ rules and
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 7b4dca61f70dc..7a933aeada62b 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4049,6 +4049,27 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
break;
}
+ case Builtin::BIcoop_mat_load:
+ return BuiltinCoopMatrixLoad(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_store:
+ return BuiltinCoopMatrixStore(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_mulAdd:
+ return BuiltinCoopMatrixMulAdd(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ return BuiltinCoopMatrixBinaryOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_mul:
+ return BuiltinCoopMatrixScalarOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_neg:
+ return BuiltinCoopMatrixScalarUnaryOp(TheCall, TheCallResult);
+
case Builtin::BI__builtin_matrix_transpose:
return BuiltinMatrixTranspose(TheCall, TheCallResult);
@@ -17296,6 +17317,270 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
return false;
}
+// Check coop_mat_load/store buffer pointer.
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
+ unsigned PtrArgIdx) {
+ bool ArgError = false;
+ Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
+ ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
+ if (PtrConv.isInvalid())
+ return true;
+ PtrExpr = PtrConv.get();
+ TheCall->setArg(PtrArgIdx, PtrExpr);
+
+ auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
+ QualType ElementTy;
+ if (!PtrTy) {
+ ArgError = true;
+ } else {
+ ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
+ if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ ArgError = true;
+ }
+ }
+
+ if (ArgError) {
+ Diag(PtrExpr->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+ << PtrArgIdx + 1 << 0 << /* pointer to element ty */ 5 << /* no fp */ 0
+ << PtrExpr->getType();
+ }
+
+ return ArgError;
+}
+
+// Check coop_mat_load/store matrix element has same type with buffer pointer.
+void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
+ auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
+ if (!MTy) {
+ Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ return;
+ }
+
+ assert(isa<PointerType>(BufferType));
+ auto *PTy = BufferType->castAs<PointerType>();
+
+ if (MTy->getElementType().getUnqualifiedType() !=
+ PTy->getPointeeType().getUnqualifiedType())
+ Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+}
+
+void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
+
+ FunctionDecl *F = call->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (Fname->isStr("coop_mat_load"))
+ CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
+ MatrixLoc);
+}
+
+// Check coop_mat_load/store layout argument
+bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
+ bool ArgError = false;
+ DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LayoutExpr);
+ if (DR) {
+ const auto *ECDHS = dyn_cast<EnumConstantDecl>(DR->getDecl());
+ if (ECDHS) {
+ if (ECDHS->getInitVal() != 0 && ECDHS->getInitVal() != 1)
+ ArgError = true;
+ } else
+ ArgError = true;
+ } else
+ ArgError = true;
+
+ if (ArgError)
+ Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+
+ return ArgError;
+}
+
+ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
+ return ExprError();
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
+ return ExprError();
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 4))
+ return ExprError();
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ return ExprError();
+ CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
+ return ExprError();
+ return CallResult;
+}
+
+void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+ FunctionDecl *F = TheCall->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (!Fname->isStr("coop_mat_mulAdd"))
+ return;
+
+ auto MC = TheCall->getArg(2);
+ auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = MC->getType()->getAs<CooperativeMatrixType>();
+ auto Loc = TheCall->getBeginLoc();
+
+ if (!MOutTy)
+ Diag(Loc, diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!MOutTy || !M2Ty)
+ return;
+
+ if (MOutTy->getUse() != 2)
+ Diag(Loc, diag::err_coop_matrix_useACC);
+
+ if (MOutTy->getElementType().getUnqualifiedType() !=
+ M2Ty->getElementType().getUnqualifiedType())
+ Diag(Loc, diag::err_coop_matrix_element_type);
+
+ if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+}
+
+bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc) {
+ auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
+ auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
+ if (!M0Ty)
+ Diag(ALoc, diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(BLoc, diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty)
+ return true;
+
+ if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+
+ if (M0Ty->getUse() != M1Ty->getUse()) {
+ Diag(ALoc, diag::err_coop_matrix_use_type);
+ return true;
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType()) {
+ Diag(ALoc, diag::err_coop_matrix_element_type);
+ return true;
+ }
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+
+ CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
+ Arg1->getBeginLoc());
+
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
+ if (ATy->isIntegerType() && CTy->isIntegerType())
+ return true;
+ if (ATy->isFloatingType() && CTy->isFloatingType())
+ return true;
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ Expr *Arg2 = TheCall->getArg(2);
+
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto *M1Ty = Arg1->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = Arg2->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+ auto Loc1 = Arg1->getBeginLoc();
+
+ if (!M0Ty)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty || !M2Ty)
+ return ExprError();
+
+ if (M0Ty->getUse() != 0)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ if (M1Ty->getUse() != 1)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ if (M2Ty->getUse() != 2)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+
+ if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
+ M2Ty->getElementType()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+
+ if (M0Ty->getNumRows() != M2Ty->getNumRows())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
+ (M0Ty->getNumColumns() != M1Ty->getNumRows()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
@@ -17306,8 +17591,9 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
- if (!MType) {
+ auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
+ auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
+ if (!ConstMType && !CoopMType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17316,11 +17602,23 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- QualType ResultType = Context.getConstantMatrixType(
- MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
+ if (ConstMType) {
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
+ if (CoopMType) {
+ QualType ResultType = Context.getCooperativeMatrixType(
+ CoopMType->getElementType(),CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
+
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index d6114ccbae7fe..a91b5a7b150c0 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14048,6 +14048,25 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
+// Return true if RHSExpr is a cooperative matrix builtin call.
+bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ if (!call)
+ return false;
+ FunctionDecl *F = call->getDirectCallee();
+ if (!F)
+ return false;
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ if (!Fname)
+ return false;
+ if (Fname->getName().starts_with("coop_mat") &&
+ !Fname->getName().starts_with("coop_mat_length"))
+ return true;
+
+ return false;
+}
+
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
llvm::scope_exit ResetDeclForInitializer([this]() {
if (!this->ExprEvalContexts.empty())
@@ -14566,6 +14585,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
+
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (!VDecl->getType()->isMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
}
void Sema::ActOnInitializerError(Decl *D) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c93efeb928c56..0e28ee5c7a0f4 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4635,6 +4635,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::TemplateSpecialization:
@@ -7951,6 +7952,16 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
+ if (srcTy->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns();
+ }
+
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -11294,12 +11305,14 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
if (!IsDiv &&
- (LHSTy->isConstantMatrixType() || RHSTy->isConstantMatrixType()))
+ (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+ if (IsDiv && LHSTy->isCooperativeMatrixType() && RHSTy->isArithmeticType())
+ return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
@@ -11701,6 +11714,15 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -11848,6 +11870,15 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -13873,6 +13904,32 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ }
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -15592,6 +15649,47 @@ static bool needsConversionOfHalfVec(bool OpRequiresConversion, ASTContext &Ctx,
return HasVectorOfHalfType(E0) && (!E1 || HasVectorOfHalfType(E1));
}
+ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Add:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
+ Args);
+ case BO_Sub:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
+ Args);
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
+ Args);
+ case BO_Div:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_binary_operator));
+}
+
+ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_scalar_operator));
+}
+
ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
BinaryOperatorKind Opc, Expr *LHSExpr,
Expr *RHSExpr, bool ForFoldExpression) {
@@ -15628,6 +15726,17 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
if (getLangOpts().OpenCL) {
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
+ // Cooperative matrix support
+ if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ // Check matrix types for assignment.
+ if (BO_Assign == Opc) {
+ if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
+ RHSExpr->getBeginLoc()))
+ return ExprError();
+ } else
+ return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
@@ -15655,6 +15764,18 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (!LHSExpr->getType()->isMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
+ return ExprError();
+ }
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ assert(call);
+ call->setType(LHSExpr->getType());
+ CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ call);
+ CheckCoopMatrixMatMulOutput(call);
+ }
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
LHS.get()->getObjectKind() != OK_ObjCProperty) {
@@ -16314,6 +16435,10 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
+ if (Opc == UO_Minus && Ty->isMatrixType()) {
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
+ {InputExpr});
+ }
// The only legal unary operation for atomics is '&'.
if ((Opc != UO_AddrOf && Ty->isAtomicType()) ||
// OpenCL special types - image, sampler, pipe, and blocks are to be used
@@ -16392,7 +16517,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
Opc == UO_Plus && resultType->isPointerType())
break;
-
+ else if (getLangOpts().OpenCL && Opc == UO_Minus &&
+ resultType->isCooperativeMatrixType())
+ break;
return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
<< resultType << Input.get()->getSourceRange());
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 319c228bad3b0..661d47b68a4ca 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3265,6 +3265,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BitInt:
break;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 4f7e59ea43ef1..bb973b8107bef 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6361,6 +6361,11 @@ bool UnnamedLocalNoLinkageFinder::VisitConstantMatrixType(
return Visit(T->getElementType());
}
+bool UnnamedLocalNoLinkageFinder::VisitCooperativeMatrixType(
+ const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+}
+
bool UnnamedLocalNoLinkageFinder::VisitFunctionProtoType(
const FunctionProtoType* T) {
for (const auto &A : T->param_types()) {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index b66152f2d971d..c4f2747a95599 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2448,6 +2448,30 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
&DependentSizedMatrixType::getColumnExpr);
}
+ // (clang extension)
+ //
+ // T __attribute__((coop_mat_type(<integral constant>,
+ // <integral constant>,
+ // <integral constant>,
+ // <integral constant>)))
+ case Type::CooperativeMatrix: {
+ const auto *MP = P->castAs<CooperativeMatrixType>(),
+ *MA = A->getAs<CooperativeMatrixType>();
+ if (!MA)
+ return TemplateDeductionResult::NonDeducedMismatch;
+
+ // Check that the dimensions are the same
+ if (MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns()) {
+ return TemplateDeductionResult::NonDeducedMismatch;
+ }
+ // Perform deduction on element types.
+ return DeduceTemplateArgumentsByTypeMatch(
+ S, TemplateParams, MP->getElementType(), MA->getElementType(), Info,
+ Deduced, TDF, degradeCallPartialOrderingKind(POK),
+ /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
+ }
+
// (clang extension)
//
// T __attribute__(((address_space(N))))
@@ -7075,6 +7099,13 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *MatType = cast<CooperativeMatrixType>(T);
+ MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
+ Depth, Used);
+ break;
+ }
+
case Type::DependentSizedMatrix: {
const DependentSizedMatrixType *MatType = cast<DependentSizedMatrixType>(T);
MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 30f34c711ab28..6e72625f6ac84 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2495,9 +2495,11 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc) {
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc, Expr *ScopeExpr,
+ Expr *UseExpr, bool IsCoopMat) {
+ if (!IsCoopMat)
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2578,6 +2580,26 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
+ if (IsCoopMat) {
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr)
+ << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
@@ -5941,6 +5963,23 @@ static void fillAtomicQualLoc(AtomicTypeLoc ATL, const DeclaratorChunk &Chunk) {
ATL.setParensRange(SourceRange());
}
+static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
+ const ParsedAttributesView &Attrs) {
+ for (const ParsedAttr &AL : Attrs) {
+ if (AL.getKind() == ParsedAttr::AT_CoopMatrixType) {
+ MTL.setAttrNameLoc(AL.getLoc());
+ MTL.setAttrScopeOperand(AL.getArgAsExpr(0));
+ MTL.setAttrRowOperand(AL.getArgAsExpr(1));
+ MTL.setAttrColumnOperand(AL.getArgAsExpr(2));
+ MTL.setAttrUseOperand(AL.getArgAsExpr(3));
+ MTL.setAttrOperandParensRange(SourceRange());
+ return;
+ }
+ }
+
+ llvm_unreachable("no matrix_type attribute found at the expected location!");
+}
+
namespace {
class TypeSpecLocFiller : public TypeLocVisitor<TypeSpecLocFiller> {
Sema &SemaRef;
@@ -6334,6 +6373,9 @@ namespace {
void VisitMatrixTypeLoc(MatrixTypeLoc TL) {
fillMatrixTypeLoc(TL, Chunk.getAttrs());
}
+ void VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
+ fillCooperativeMatrixTypeLoc(TL, Chunk.getAttrs());
+ }
void VisitTypeLoc(TypeLoc TL) {
llvm_unreachable("unsupported TypeLoc kind in declarator!");
@@ -8939,23 +8981,28 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S) {
- if (!S.getLangOpts().MatrixTypes) {
+ Sema &S, bool IsCoopMat = false) {
+ if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- if (Attr.getNumArgs() != 2) {
+ unsigned int NumAttrs = IsCoopMat ? 4 : 2;
+ if (Attr.getNumArgs() != NumAttrs) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << 2;
+ << Attr << NumAttrs;
return;
}
-
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ if (IsCoopMat) {
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
+ Scope, Use, /* IsCoopMat */ true);
+ if (!T.isNull())
+ CurType = T;
+ }
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9227,6 +9274,11 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
attr.setUsedAsTypeAttr();
break;
+ case ParsedAttr::AT_CoopMatrixType:
+ HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ attr.setUsedAsTypeAttr();
+ break;
+
case ParsedAttr::AT_WebAssemblyFuncref: {
if (!HandleWebAssemblyFuncrefAttr(state, type, attr))
attr.setUsedAsTypeAttr();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 5f9a7d53fb259..101d2683ab18b 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1047,6 +1047,12 @@ class TreeTransform {
QualType RebuildConstantMatrixType(QualType ElementType, unsigned NumRows,
unsigned NumColumns);
+ /// Build a new cooperative matrix type given the element type and
+ /// scope and use and dimensions.
+ QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use);
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
@@ -6302,6 +6308,34 @@ TreeTransform<Derived>::TransformConstantMatrixType(TypeLocBuilder &TLB,
return Result;
}
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformCooperativeMatrixType(
+ TypeLocBuilder &TLB, CooperativeMatrixTypeLoc TL) {
+ const CooperativeMatrixType *T = TL.getTypePtr();
+ QualType ElementType = getDerived().TransformType(T->getElementType());
+ if (ElementType.isNull())
+ return QualType();
+
+ QualType Result = TL.getType();
+ if (getDerived().AlwaysRebuild() || ElementType != T->getElementType()) {
+ Result = getDerived().RebuildCooperativeMatrixType(
+ ElementType, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ if (Result.isNull())
+ return QualType();
+ }
+
+ CooperativeMatrixTypeLoc NewTL = TLB.push<CooperativeMatrixTypeLoc>(Result);
+ NewTL.setAttrNameLoc(TL.getAttrNameLoc());
+ NewTL.setAttrOperandParensRange(TL.getAttrOperandParensRange());
+ NewTL.setAttrScopeOperand(TL.getAttrScopeOperand());
+ NewTL.setAttrRowOperand(TL.getAttrRowOperand());
+ NewTL.setAttrColumnOperand(TL.getAttrColumnOperand());
+ NewTL.setAttrUseOperand(TL.getAttrUseOperand());
+
+ return Result;
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::TransformDependentSizedMatrixType(
TypeLocBuilder &TLB, DependentSizedMatrixTypeLoc TL) {
@@ -18165,6 +18199,14 @@ QualType TreeTransform<Derived>::RebuildConstantMatrixType(
NumColumns);
}
+template <typename Derived>
+QualType TreeTransform<Derived>::RebuildCooperativeMatrixType(
+ QualType ElementType, unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use) {
+ return SemaRef.Context.getCooperativeMatrixType(ElementType, Scope, NumRows,
+ NumColumns, Use);
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::RebuildDependentSizedMatrixType(
QualType ElementType, Expr *RowExpr, Expr *ColumnExpr,
>From e06c3b7d1185812939d3ca3121224e99d86e6caf Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:56 -0400
Subject: [PATCH 05/17] Patch 5 - Emit LLVM IR and intrinsics
---
clang/include/clang/Basic/Builtins.td | 67 +++++++
clang/lib/CodeGen/CGBuiltin.cpp | 277 ++++++++++++++++++++++++++
clang/lib/CodeGen/CGDebugInfo.cpp | 29 +++
clang/lib/CodeGen/CGDebugInfo.h | 1 +
clang/lib/CodeGen/CGExpr.cpp | 61 ++++++
clang/lib/CodeGen/CGExprScalar.cpp | 14 ++
clang/lib/CodeGen/CodeGenFunction.cpp | 2 +
clang/lib/CodeGen/CodeGenFunction.h | 9 +
clang/lib/CodeGen/CodeGenTBAA.cpp | 2 +-
clang/lib/CodeGen/CodeGenTypes.cpp | 33 ++-
clang/lib/CodeGen/ItaniumCXXABI.cpp | 2 +
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +
clang/lib/CodeGen/QualTypeMapper.h | 1 +
13 files changed, 502 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 49fe879c6add1..cfa9e0801b0f1 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,6 +1931,73 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
+// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+def CoopMatLoad : Builtin {
+ let Spellings = ["coop_mat_load"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatStore : Builtin {
+ let Spellings = ["coop_mat_store"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatMulAdd : Builtin {
+ let Spellings = ["coop_mat_mulAdd"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryAdd : Builtin {
+ let Spellings = ["coop_mat_binary_add"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinarySub : Builtin {
+ let Spellings = ["coop_mat_binary_sub"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryMul : Builtin {
+ let Spellings = ["coop_mat_binary_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryDiv : Builtin {
+ let Spellings = ["coop_mat_binary_div"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarMul : Builtin {
+ let Spellings = ["coop_mat_scalar_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarNeg : Builtin {
+ let Spellings = ["coop_mat_scalar_neg"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatInit : Builtin {
+ let Spellings = ["coop_mat_init"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatLength : Builtin {
+ let Spellings = ["coop_mat_length"];
+ let Attributes = [NoThrow, Const, CustomTypeChecking];
+ let Prototype = "unsigned int(...)";
+}
+
// "Overloaded" Atomic operator builtins. These are overloaded to support data
// types of i8, i16, i32, i64, and i128. The front-end sees calls to the
// non-suffixed version of these (which has a bogus type) and transforms them to
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index f4598a8a54e1b..486b7fd554ee1 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2756,6 +2756,54 @@ static void ClearPadding(CodeGenFunction &CGF, Address Src,
}
}
+static std::string getSPIRVBuiltinName(unsigned BuiltinID,
+ bool IsFloat = false) {
+ switch (BuiltinID) {
+ case Builtin::BIcoop_mat_load:
+ return "__spirv_CooperativeMatrixLoadKHR";
+ case Builtin::BIcoop_mat_store:
+ return "__spirv_CooperativeMatrixStoreKHR";
+ case Builtin::BIcoop_mat_mulAdd:
+ return "__spirv_CooperativeMatrixMulAddKHR";
+ case Builtin::BIcoop_mat_binary_add:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFAdd" : "__spirv_IAdd";
+ case Builtin::BIcoop_mat_binary_sub:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFSub" : "__spirv_ISub";
+ case Builtin::BIcoop_mat_binary_mul:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFMul" : "__spirv_IMul";
+ case Builtin::BIcoop_mat_binary_div:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFDiv" : "__spirv_IDiv";
+ case Builtin::BIcoop_mat_scalar_mul:
+ return "__spirv_CooperativeMatrixScalarMulKHR";
+ case Builtin::BIcoop_mat_scalar_neg:
+ return "__spirv_CooperativeMatrixScalarNeg";
+ case Builtin::BIcoop_mat_init:
+ return "__spirv_CompositeConstruct";
+ case Builtin::BIcoop_mat_length:
+ return "__spirv_CooperativeMatrixLengthKHR";
+ }
+ assert(0 && "Unexpected Builtin");
+ return "";
+}
+
+static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
+ CodeGenModule &CGM,
+ const CooperativeMatrixType *MTy) {
+ llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
+ MTy->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
+} // namespace
+
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
@@ -4608,6 +4656,235 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(Result);
}
+ case Builtin::BIcoop_mat_load: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address SrcAddr = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(SrcAddr, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_load without coop_mat output operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Layout = EmitScalarExpr(E->getArg(1));
+ auto Stride = EmitScalarExpr(E->getArg(2));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_store: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address Dst = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(Dst, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_store without coop_mat input operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Arg0 = EmitScalarExpr(E->getArg(1));
+ auto Layout = EmitScalarExpr(E->getArg(2));
+ auto Stride = EmitScalarExpr(E->getArg(3));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_mulAdd: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_mulAdd without coop_mat output operand");
+
+ auto Arg0 = E->getArg(0);
+ auto Arg1 = E->getArg(1);
+ auto Arg2 = E->getArg(2);
+ auto MA = EmitScalarExpr(Arg0);
+ auto MB = EmitScalarExpr(Arg1);
+ auto MC = EmitScalarExpr(Arg2);
+ const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
+ const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MATy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *CType = getTargetExtType(*this, CGM, MCTy);
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, CType, Int1Ty}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ llvm::Type *ElTy = ConvertType(MTy->getElementType());
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name = (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_mul: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = Arg1->getType();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_neg: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_init: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ auto Init = EmitScalarExpr(E->getArg(0));
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Init->getType()}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(InitFn, {Init});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_length: {
+ const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ ConvertType(E->getType()), {AType}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
case Builtin::BI__builtin_masked_load:
case Builtin::BI__builtin_masked_expand_load: {
llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 2c7af395c4562..35f7213d1d758 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3840,6 +3840,33 @@ llvm::DIType *CGDebugInfo::CreateType(const ConstantMatrixType *Ty,
return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
}
+llvm::DIType *CGDebugInfo::CreateType(const CooperativeMatrixType *Ty,
+ llvm::DIFile *Unit) {
+ // FIXME: Create another debug type for matrices
+ // For the time being, it treats it like a nested ArrayType.
+
+ llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
+ uint64_t Size = CGM.getContext().getTypeSize(Ty);
+ uint32_t Align = getTypeAlignIfRequired(Ty, CGM.getContext());
+
+ // Create ranges for both dimensions.
+ llvm::SmallVector<llvm::Metadata *, 2> Subscripts;
+ auto *ColumnCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumColumns()));
+ auto *RowCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumRows()));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ ColumnCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ RowCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
+ return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
+}
+
llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
uint64_t Size;
uint32_t Align;
@@ -4299,6 +4326,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
return CreateType(cast<VectorType>(Ty), Unit);
case Type::ConstantMatrix:
return CreateType(cast<ConstantMatrixType>(Ty), Unit);
+ case Type::CooperativeMatrix:
+ return CreateType(cast<CooperativeMatrixType>(Ty), Unit);
case Type::ObjCObjectPointer:
return CreateType(cast<ObjCObjectPointerType>(Ty), Unit);
case Type::ObjCObject:
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 8a46e3f0e60bb..6dc3789ed9eb0 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -241,6 +241,7 @@ class CGDebugInfo {
llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ConstantMatrixType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const CooperativeMatrixType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index cba81de9d10dd..25800c4a42ef1 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7523,3 +7523,64 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
}
+
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
+ llvm::Value *ScalarVal,
+ QualType CoopMatQTy)
+{
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(
+ VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(
+ VecTy->getElementCount(),
+ ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
+ BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy)
+{
+ auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
+
+ switch (Opcode) {
+ case BO_Add:
+ return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
+ : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+
+ case BO_Sub:
+ return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
+ : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+
+ case BO_Mul:
+ // mat * mat → element-wise FMul/IMul
+ // mat * scalar → handled separately (see scalar path below)
+ if (!RHS->getType()->isVectorTy()) {
+ // mat * scalar: broadcast scalar then element-wise mul
+ llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
+ return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
+ : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ }
+ return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
+ : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+
+ case BO_Div:
+ if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+
+ default:
+ llvm_unreachable("Unsupported cooperative matrix binary op");
+ }
+}
\ No newline at end of file
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 8a1dd776118f5..954a123143e21 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -954,6 +954,12 @@ class ScalarExprEmitter
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
+ if (E->getType()->isCooperativeMatrixType()) { \
+ return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
+ CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), \
+ E->getType()); \
+ } \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
if (result && !promotionTy.isNull()) \
@@ -3691,6 +3697,14 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
+ QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ if (CompTy->isFloatingType())
+ return Builder.CreateFNeg(Val, "coopmat.fneg");
+ else
+ return Builder.CreateNeg(Val, "coopmat.ineg");
+ }
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 119aebb673789..6562bcaf1eeda 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -277,6 +277,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -2569,6 +2570,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 7bdc79d86ea0a..7723344bb11d6 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5133,6 +5133,15 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
+ /// Helper function for EmitCoopMatBinaryOp
+ llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 1854df7c7c0f1..ecd32fb65e9a1 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -331,7 +331,7 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert(isa<ConstantMatrixType>(Ty) &&
+ assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 99ead1295bc58..3f10aca48fb02 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -120,7 +120,21 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
-
+ if (T->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent a Coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+ return CoopMatType;
+ }
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -688,6 +702,23 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
MT->getNumRows() * MT->getNumColumns());
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+
+ ResultType = CoopMatType;
+ break;
+ }
case Type::FunctionNoProto:
case Type::FunctionProto:
ResultType = ConvertFunctionTypeInternal(T);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index c17813140b10f..32161ab146956 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -4018,6 +4018,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -4291,6 +4292,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 212a138f9b7b7..54d680faaf632 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -108,6 +108,12 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
MT->getNumRows() * MT->getNumColumns(),
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(QT);
+ return Builder.getArrayType(convertType(MT->getElementType()),
+ MT->getNumRows() * MT->getNumColumns(),
+ ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ }
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
case Type::BitInt: {
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index 35876f44f3aba..76073e980d576 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -47,6 +47,7 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
+ const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
>From 535cbd1439ba70394a813ad0807294e4fbfb44ae Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:20:46 -0400
Subject: [PATCH 06/17] Add LIT tests and fix minor build issues
---
clang/include/clang/AST/TypeBase.h | 8 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 94 +++++++++++++++
.../test/Preprocessor/coop-mat-opencl-ext.cl | 63 ++++++++++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 66 ++++++++++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 60 +++++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 83 +++++++++++++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 114 ++++++++++++++++++
clang/tools/libclang/CIndex.cpp | 1 +
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ++++++
9 files changed, 524 insertions(+), 4 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen.cl
create mode 100644 clang/test/Preprocessor/coop-mat-opencl-ext.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-ast-utils.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema-neg.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-type-infra.cl
create mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 4b3fba1f9934c..560f65a68b8f6 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4632,14 +4632,14 @@ class CooperativeMatrixType final : public MatrixType {
/// Return true if \p Scope is valid
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
}
/// Return true if \p Use is valid
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
}
/// Returns the maximum number of elements per dimension.
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
new file mode 100644
index 0000000000000..58b7a882fdc87
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -0,0 +1,94 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+//
+// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
+// SPIR-V intrinsic names.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
+// TargetExtType (visible in the call signature).
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// ---------------------------------------------------------------------------
+kernel void test_store(__global float *ptr, MatA_t a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
+
+// ---------------------------------------------------------------------------
+// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a; MatB_t b; MatC_t c; MatC_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+
+// ---------------------------------------------------------------------------
+// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// ---------------------------------------------------------------------------
+kernel void test_binary_add(__global float *ptr) {
+ MatA_t a; MatA_t b; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a + b;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_binary_add
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
+
+// ---------------------------------------------------------------------------
+// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// ---------------------------------------------------------------------------
+kernel void test_scalar_mul(__global float *ptr, float s) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a * s;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+
+// ---------------------------------------------------------------------------
+// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// ---------------------------------------------------------------------------
+kernel void test_unary_neg(__global float *ptr) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = -a;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_unary_neg
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarNeg
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
new file mode 100644
index 0000000000000..7181efbe7e2a8
--- /dev/null
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -0,0 +1,63 @@
+// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+//
+// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
+//
+// Tests: extension macro is predefined when enabled, extension can be
+// explicitly enabled/disabled via pragma, all four enum types and
+// their constants are visible under -finclude-default-header.
+
+// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E -dM %s \
+// RUN: | FileCheck %s --check-prefix=EXT
+
+// EXT: cl_ext_kernel_cooperative_matrix
+
+// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E %s \
+// RUN: | FileCheck %s --check-prefix=NOEXT
+
+// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+
+// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+void test_enum_constants(void) {
+ // coop_matrix_scope_t
+ coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
+ (void)s;
+
+ // coop_matrix_use_t
+ coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
+ coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
+ coop_matrix_use_t u2 = CLK_COOPERATIVE_MATRIX_ACCUMULATOR;
+ (void)u0; (void)u1; (void)u2;
+
+ // coop_matrix_layout_t
+ coop_matrix_layout_t l0 = CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR;
+ coop_matrix_layout_t l1 = CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR;
+ (void)l0; (void)l1;
+
+ // coop_matrix_operands_t
+ coop_matrix_operands_t op = CLK_COOPERATIVE_MATRIX_OPERAND_NONE;
+ (void)op;
+}
+
+// ── 2d. Enum constant values match the spec ──────────────────────────────────
+void test_enum_values(void) {
+ _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR == 0, "row major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR== 1, "col major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_OPERAND_NONE == 0, "operand none");
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
new file mode 100644
index 0000000000000..c6230eb94531f
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -0,0 +1,66 @@
+// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
+//
+// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
+// ExprConstant, ItaniumMangle, MicrosoftMangle,
+// ASTReader/ASTWriter TypeLoc serialisation.
+//
+// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefix=MANGLE
+//
+// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch \
+// RUN: -ast-dump %t.pch_probe.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+//
+// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
+kernel void test_mangling(MatA_t a) { (void)a; }
+
+// MANGLE: @{{.*}}test_mangling{{.*}}(
+
+// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
+// must still carry a coop_mat parameter type.
+// PCH: FunctionDecl {{.*}} pch_probe
+// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
+
+// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_structural_equiv(void) {
+ MatA_t *p = 0;
+ MatA_alias *q = p; // same canonical type
+ (void)q;
+}
+
+// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
+// integer, float, pointer …). Using __builtin_classify_type on it
+// compiles without error; result is 0 (no_type_class).
+void test_expr_constant(MatA_t a) {
+ int cls = __builtin_classify_type(a);
+ (void)cls;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
new file mode 100644
index 0000000000000..f8080448b6df5
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -0,0 +1,60 @@
+// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+//
+// Patch 4: Sema — negative tests for diagnostic paths.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 2. Invalid use value (99 is not 0/1/2)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 3. Mismatched element types in coop_mat_mulAdd
+// a/b are float matrices, c is an int matrix — should fire element type
+// mismatch diagnostic.
+// Use the two-step declare-then-assign pattern so that the coop_mat_load
+// fixup path fires correctly and we only get the intended mulAdd error.
+// ---------------------------------------------------------------------------
+kernel void test_muladd_type_mismatch(__global float *fptr,
+ __global int *iptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+ a = coop_mat_load(fptr, ROW_MAJOR, 16);
+ b = coop_mat_load(fptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+
+ MatC_int_t result;
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
+// This intentionally uses single-step initialization because the error
+// fires precisely when the LHS is NOT a coop mat type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_assignment(__global float *ptr) {
+ float bad;
+ bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
+ (void)bad;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
new file mode 100644
index 0000000000000..cec1f278b0037
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -0,0 +1,83 @@
+// clang/test/SemaOpenCL/coop-mat-sema.cl
+//
+// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
+// builtin validation, expr handling, TreeTransform.
+//
+// Valid code -- expected-no-diagnostics
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
+// match the LHS variable type. Declare first, then assign so that
+// Sema's IsCoopMatrixBuiltin path fires correctly.
+// ---------------------------------------------------------------------------
+kernel void test_load_store(__global float *ptr,
+ __global float *out_ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ coop_mat_store(out_ptr, a, ROW_MAJOR, 16);
+}
+
+// ---------------------------------------------------------------------------
+// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ MatC_t result;
+ result = coop_mat_mulAdd(a, b, c);
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4d. Binary element-wise operators (+, -)
+// ---------------------------------------------------------------------------
+void test_binary_ops(MatA_t a, MatA_t b) {
+ MatA_t r_add = a + b;
+ MatA_t r_sub = a - b;
+ (void)r_add; (void)r_sub;
+}
+
+// ---------------------------------------------------------------------------
+// 4e. Scalar multiply operator
+// ---------------------------------------------------------------------------
+void test_scalar_ops(MatA_t a, float s) {
+ MatA_t r = a * s;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4f. Unary minus
+// ---------------------------------------------------------------------------
+void test_unary_minus(MatA_t a) {
+ MatA_t r = -a;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
+// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// ---------------------------------------------------------------------------
+kernel void test_assignment(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
new file mode 100644
index 0000000000000..2e6700973e611
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -0,0 +1,114 @@
+// clang/test/SemaOpenCL/coop_mat_type_infra.cl
+//
+// Patch 1: CooperativeMatrixType AST node and type infrastructure.
+//
+// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
+// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
+// RecursiveASTVisitor traversal, mergeTypes compatibility,
+// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -ast-dump %s \
+// RUN: | FileCheck %s --check-prefix=AST
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+
+// ---------------------------------------------------------------------------
+// Enum constants (from opencl-c-base.h via -finclude-default-header)
+// ---------------------------------------------------------------------------
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+// ---------------------------------------------------------------------------
+// 1. Basic type construction — four use roles, float element type.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_float16x16;
+typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
+
+// AST: TypedefDecl {{.*}} MatA_float16x16
+// AST: TypedefDecl {{.*}} MatB_float16x16
+// AST: TypedefDecl {{.*}} MatC_float16x16
+// AST: TypedefDecl {{.*}} MatA_int8x8
+
+// ---------------------------------------------------------------------------
+// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// ---------------------------------------------------------------------------
+void test_type_spelling(void) {
+ MatA_float16x16 a;
+ MatB_float16x16 b;
+ MatC_float16x16 c;
+ MatA_int8x8 d;
+}
+
+// AST: VarDecl {{.*}} a {{.*}}coop_mat(
+// AST: VarDecl {{.*}} b {{.*}}coop_mat(
+// AST: VarDecl {{.*}} c {{.*}}coop_mat(
+// AST: VarDecl {{.*}} d {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
+// float(4B)*16*16 = 1024 B = 8192 bits
+// int(4B)*8*8 = 256 B = 2048 bits
+// ---------------------------------------------------------------------------
+void test_sizeof(void) {
+ _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
+ _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
+}
+
+// ---------------------------------------------------------------------------
+// 4. Parameter / return type — type preserved through function boundary.
+// ---------------------------------------------------------------------------
+MatA_float16x16 test_param_return(MatA_float16x16 in) {
+ return in;
+}
+
+// AST: FunctionDecl {{.*}} test_param_return
+// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 5. TypeLoc operand traversal — all four operand slots populated.
+// ---------------------------------------------------------------------------
+void test_typeloc_operands(void) {
+ float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
+ (void)local_acc;
+}
+
+// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// ---------------------------------------------------------------------------
+void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
+ (void)x;
+}
+
+// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_merge_types(void) {
+ MatA_float16x16 *p = 0;
+ MatA_alias *q = p; // same canonical type — no diagnostic
+ (void)q;
+}
+
+// ---------------------------------------------------------------------------
+// 8. PCH serialisation round-trip.
+// ---------------------------------------------------------------------------
+// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 39e9e89b1ff00..3e7320deacdbb 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1909,6 +1909,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
new file mode 100644
index 0000000000000..c4e206b76b760
--- /dev/null
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -0,0 +1,39 @@
+; llvm/test/Verifier/coop_mat_verifier_fix.ll
+;
+; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
+;
+; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
+; ((A && B) <= 64). After the fix the parentheses are explicit.
+;
+; This test verifies that a valid @llvm.experimental.noalias.scope.decl
+; call with a 2-element scope list (no third argument) passes the verifier
+; without error, and that an invalid third argument (> 64 bits) is caught.
+;
+; ── 6a. Valid call — no third argument → verifier must accept ────────────────
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
+; RUN: llvm-as < %s | opt -passes=verify -disable-output
+;
+; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
+; RUN: llvm-as %s -o %t.bc
+; RUN: opt -passes=verify %t.bc -disable-output
+
+; VALID: @llvm.experimental.noalias.scope.decl
+
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+define void @test_no_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+; Third arg present and <= 64 bits wide — must also pass
+define void @test_valid_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain_a"}
>From 9303a8163dc25a92e133da30206c6d7e2ac132f3 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 18:39:15 -0400
Subject: [PATCH 07/17] Fix formatting issues
---
clang/include/clang/Sema/Sema.h | 15 +++---
clang/lib/CodeGen/CGBuiltin.cpp | 76 +++++++++++++++------------
clang/lib/CodeGen/CGExpr.cpp | 63 ++++++++++------------
clang/lib/CodeGen/CGExprScalar.cpp | 12 ++---
clang/lib/Sema/SemaChecking.cpp | 39 +++++++-------
clang/lib/Sema/SemaExpr.cpp | 35 ++++++------
clang/lib/Sema/TreeTransform.h | 2 +-
clang/lib/Serialization/ASTReader.cpp | 3 +-
clang/lib/Serialization/ASTWriter.cpp | 3 +-
9 files changed, 120 insertions(+), 128 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index b10a5b3f914c1..587027005c75f 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3062,19 +3062,16 @@ class Sema final : public SemaBase {
// Coop matrix handling.
void CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc, CallExpr *call);
+ SourceLocation MatrixLoc, CallExpr *call);
void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc);
+ QualType BufferType,
+ SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
- ExprResult CallResult);
- ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
- ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 486b7fd554ee1..6a296cb98d216 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2794,7 +2794,7 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
llvm::Type *Tys[] = {ElTy};
// Unsigned arguments for TargetExtType
unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
+ MTy->getUse()};
// Create a TargetExtType to represent the coop matrix type
llvm::TargetExtType *RetType = llvm::TargetExtType::get(
CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
@@ -4665,8 +4665,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_load without coop_mat output operand");
+ CGM.ErrorUnsupported(E, "coop_mat_load without coop_mat output operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Layout = EmitScalarExpr(E->getArg(1));
@@ -4680,7 +4679,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LoadFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
@@ -4697,8 +4697,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_store without coop_mat input operand");
+ CGM.ErrorUnsupported(E, "coop_mat_store without coop_mat input operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Arg0 = EmitScalarExpr(E->getArg(1));
@@ -4708,11 +4707,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
- VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
+ false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee StoreFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
@@ -4723,8 +4724,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_mulAdd: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_mulAdd without coop_mat output operand");
+ CGM.ErrorUnsupported(E,
+ "coop_mat_mulAdd without coop_mat output operand");
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
@@ -4748,11 +4749,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto *AType = getTargetExtType(*this, CGM, MATy);
auto *BType = getTargetExtType(*this, CGM, MBTy);
auto *CType = getTargetExtType(*this, CGM, MCTy);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee MatMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
@@ -4766,7 +4768,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_binary_div: {
auto Arg0 = EmitScalarExpr(E->getArg(0));
auto Arg1 = EmitScalarExpr(E->getArg(1));
-
+
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
@@ -4783,13 +4785,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
-
- std::string Name = (ElTy->isIntegerTy())
- ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
- : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
- llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name =
+ (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
@@ -4815,10 +4819,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
@@ -4842,10 +4847,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarNegFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
@@ -4858,10 +4864,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Init = EmitScalarExpr(E->getArg(0));
auto *RetType = getTargetExtType(*this, CGM, MTy);
// Set function type.
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {Init->getType()}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {Init->getType()}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee InitFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(InitFn, {Init});
@@ -4873,11 +4880,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
auto *AType = getTargetExtType(*this, CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- ConvertType(E->getType()), {AType}, false);
-
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
+
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LengthFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 25800c4a42ef1..897d29a655298 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7524,44 +7524,37 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
- llvm::Value *ScalarVal,
- QualType CoopMatQTy)
-{
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
-
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(
- VecTy->getElementCount(), C);
-
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(
- VecTy->getElementCount(),
- ScalarVal,
- "coopmat.broadcast");
-}
-
-llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
- BinaryOperatorKind Opcode,
- llvm::Value *LHS,
- llvm::Value *RHS,
- QualType ResultTy)
-{
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy) {
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy) {
auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
switch (Opcode) {
case BO_Add:
return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+ : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
case BO_Sub:
return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+ : Builder.CreateSub(LHS, RHS, "coopmat.isub");
case BO_Mul:
// mat * mat → element-wise FMul/IMul
@@ -7570,14 +7563,16 @@ llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
// mat * scalar: broadcast scalar then element-wise mul
llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
}
return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+ : Builder.CreateMul(LHS, RHS, "coopmat.imul");
case BO_Div:
- if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ if (IsFloat)
+ return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned)
+ return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
default:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 954a123143e21..68d1b4dea16dc 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -955,10 +955,9 @@ class ScalarExprEmitter
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
if (E->getType()->isCooperativeMatrixType()) { \
- return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
- CGF.EmitScalarExpr(E->getLHS()), \
- CGF.EmitScalarExpr(E->getRHS()), \
- E->getType()); \
+ return CGF.EmitCoopMatBinaryOp( \
+ E->getOpcode(), CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), E->getType()); \
} \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
@@ -3699,11 +3698,12 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ QualType CompTy =
+ E->getType()->getAs<CooperativeMatrixType>()->getElementType();
if (CompTy->isFloatingType())
return Builder.CreateFNeg(Val, "coopmat.fneg");
else
- return Builder.CreateNeg(Val, "coopmat.ineg");
+ return Builder.CreateNeg(Val, "coopmat.ineg");
}
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 7a933aeada62b..9101ba7a27c4d 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17318,8 +17318,7 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
}
// Check coop_mat_load/store buffer pointer.
-bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
- unsigned PtrArgIdx) {
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
bool ArgError = false;
Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
@@ -17350,8 +17349,8 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
// Check coop_mat_load/store matrix element has same type with buffer pointer.
void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc) {
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
@@ -17367,8 +17366,8 @@ void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
}
void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc,
- CallExpr *call) {
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
FunctionDecl *F = call->getDirectCallee();
assert(F);
@@ -17377,7 +17376,7 @@ void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
assert(Fname);
if (Fname->isStr("coop_mat_load"))
CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ MatrixLoc);
}
// Check coop_mat_load/store layout argument
@@ -17401,7 +17400,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
}
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
@@ -17420,7 +17419,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
return CallResult;
@@ -17459,7 +17458,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
- SourceLocation BLoc) {
+ SourceLocation BLoc) {
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
@@ -17488,7 +17487,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
}
ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17496,7 +17495,7 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ Arg1->getBeginLoc());
TheCall->setType(Arg0->getType());
@@ -17512,7 +17511,7 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
}
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
@@ -17560,7 +17559,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17571,7 +17570,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
return ExprError();
@@ -17604,17 +17603,17 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// matrix type.
if (ConstMType) {
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
}
if (CoopMType) {
QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(),CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
+ CoopMType->getElementType(), CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 0e28ee5c7a0f4..8c023bcade5a6 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -11304,8 +11304,7 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
if (LHSTy->isSveVLSBuiltinType() || RHSTy->isSveVLSBuiltinType())
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
- if (!IsDiv &&
- (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
+ if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
@@ -13905,10 +13904,11 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return QualType();
if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
+ RHS.get()->getType()->isCooperativeMatrixType()) {
auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a matrix");
if (LHSMatType && RHSMatType) {
if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
return InvalidOperands(Loc, LHS, RHS);
@@ -13919,14 +13919,14 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
RHS.get()->getType().getUnqualifiedType());
QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
+ RHSELTy = RHSMatType->getElementType();
if (!Context.hasSameType(LHSELTy, RHSELTy))
return InvalidOperands(Loc, LHS, RHS);
return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
- LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
- LHSMatType->getUse());
+ Context.getCommonSugaredType(LHSELTy, RHSELTy),
+ LHSMatType->getScope(), LHSMatType->getNumRows(),
+ RHSMatType->getNumColumns(), LHSMatType->getUse());
}
}
@@ -15657,17 +15657,13 @@ ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Add:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add, Args);
case BO_Sub:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub, Args);
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul, Args);
case BO_Div:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div, Args);
default:
break;
}
@@ -15682,8 +15678,7 @@ ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul, Args);
default:
break;
}
@@ -15731,7 +15726,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
- RHSExpr->getBeginLoc()))
+ RHSExpr->getBeginLoc()))
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
@@ -15773,7 +15768,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
assert(call);
call->setType(LHSExpr->getType());
CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
+ call);
CheckCoopMatrixMatMulOutput(call);
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 101d2683ab18b..125790ab5ec39 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1052,7 +1052,7 @@ class TreeTransform {
QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
unsigned NumRows, unsigned NumColumns,
unsigned Use);
-
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index bbb31f8f26aac..e29056a98cb3a 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7593,8 +7593,7 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
-void TypeLocReader::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
TL.setAttrOperandParensRange(readSourceRange());
TL.setAttrScopeOperand(Reader.readExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 0f14f50a1ece8..f575e6e528433 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -454,8 +454,7 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
-void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
SourceRange range = TL.getAttrOperandParensRange();
addSourceLocation(range.getBegin());
>From c1f417b99cfac1202228ec64913b6cedd162d3c7 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 15 Jul 2026 12:12:32 -0400
Subject: [PATCH 08/17] Add tests for coop_mat_init and coop_mat_length
---
.../coop-mat-init-length-codegen.cl | 112 ++++++++++++++++++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 94 +++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
new file mode 100644
index 0000000000000..b18f5617338c8
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -0,0 +1,112 @@
+// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+// ---------------------------------------------------------------------------
+// Type definitions.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init — float scalar.
+// Expected IR: call __spirv_CompositeConstruct with the scalar value,
+// returning a spirv.CooperativeMatrixKHR TargetExtType.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_float
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float
+
+kernel void test_init_float(global float *out) {
+ MatA_t a;
+ a = coop_mat_init(1.0f);
+ (void)a;
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init — integer scalar.
+// Expected IR: call __spirv_CompositeConstruct with an i32 value.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_int
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(i32
+
+kernel void test_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+ (void)m;
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init — zero initialisation.
+// Expected IR: call __spirv_CompositeConstruct with 0.0.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_zero
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+
+kernel void test_init_zero(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ (void)acc;
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length — returns unsigned int.
+// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
+// an i32 alloca.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+
+kernel void test_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ *out = len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_init then coop_mat_length — combined flow.
+// Verifies the TargetExtType produced by init flows into length unchanged.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_then_length
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_init_then_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(2.0f);
+ *out = coop_mat_length(a);
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// The call must NOT be a plain 'call' — it must be 'call spir_func'.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length_cc
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_length_cc(global unsigned int *out) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ *out = coop_mat_length(acc);
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
new file mode 100644
index 0000000000000..ab499bc1b4af7
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -0,0 +1,94 @@
+// Tests for coop_mat_init and coop_mat_length — Sema (positive).
+//
+// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
+// so arg-count and arg-type validation are not performed by Sema for these two
+// builtins. Only the positive (valid-code) path and the one assignment-target
+// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
+// tested here.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics (sections 1-6 must compile cleanly)
+
+// ---------------------------------------------------------------------------
+// Type definitions shared across all tests.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_B))) MatB_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init: initialise each use role with a float scalar.
+// Two-step (declare then assign) is required because the return type is
+// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_basic(void) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_init(1.0f);
+ b = coop_mat_init(2.0f);
+ c = coop_mat_init(0.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init: integer element type.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_chained(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ acc = coop_mat_init(1.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length: basic usage — returns unsigned int directly.
+// No two-step required for the length call itself.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_basic(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ (void)len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_length: all three use roles.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_roles(void) {
+ MatA_t a; a = coop_mat_init(0.0f);
+ MatB_t b; b = coop_mat_init(0.0f);
+ MatC_t c; c = coop_mat_init(0.0f);
+ unsigned int la = coop_mat_length(a);
+ unsigned int lb = coop_mat_length(b);
+ unsigned int lc = coop_mat_length(c);
+ (void)la; (void)lb; (void)lc;
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length: result used in arithmetic.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_arith(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int half_len = coop_mat_length(a) / 2u;
+ (void)half_len;
+}
>From 1e24cbe21fca8289a6d8fa0e783e29d9f57dad26 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 19:21:14 -0400
Subject: [PATCH 09/17] Add mangling support
Signed-off-by: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
---
clang/lib/CodeGen/CGBuiltin.cpp | 91 ++++++++-
.../coop-mat-codegen-mangling.cl | 174 ++++++++++++++++++
.../coop-mat-init-length-codegen.cl | 14 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 15 ++
4 files changed, 280 insertions(+), 14 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 6a296cb98d216..a40606ef376d8 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2802,6 +2802,62 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
return RetType;
}
+/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
+/// TargetExtType: element type + scope + rows + cols + use.
+/// E.g. target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) ->
+/// "_f32_sc3_16x16_u2"
+static std::string getCoopMatFullSuffix(llvm::Type *Ty) {
+ auto *TET = cast<llvm::TargetExtType>(Ty);
+ llvm::Type *ElemTy = TET->getTypeParameter(0);
+
+ // Element type string
+ std::string ElemStr;
+ if (ElemTy->isFloatTy())
+ ElemStr = "f32";
+ else if (ElemTy->isHalfTy())
+ ElemStr = "f16";
+ else if (ElemTy->isDoubleTy())
+ ElemStr = "f64";
+ else if (auto *IntTy = dyn_cast<llvm::IntegerType>(ElemTy))
+ ElemStr = "i" + std::to_string(IntTy->getBitWidth());
+ else
+ llvm_unreachable("Unsupported cooperative matrix element type");
+
+ // Integer parameters: scope, rows, cols, use
+ unsigned Scope = TET->getIntParameter(0);
+ unsigned Rows = TET->getIntParameter(1);
+ unsigned Cols = TET->getIntParameter(2);
+ unsigned Use = TET->getIntParameter(3);
+
+ return "_" + ElemStr + "_sc" + std::to_string(Scope) + "_" +
+ std::to_string(Rows) + "x" + std::to_string(Cols) + "_u" +
+ std::to_string(Use);
+}
+
+/// Returns a name suffix encoding the address space of a pointer type,
+/// matching OpenCL/SPIR-V address space conventions.
+/// E.g. ptr addrspace(1) -> "_global"
+/// ptr addrspace(3) -> "_local"
+/// ptr addrspace(0) -> "_private"
+/// ptr addrspace(4) -> "_generic"
+static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
+ unsigned AS = cast<llvm::PointerType>(PtrTy)->getAddressSpace();
+ switch (AS) {
+ case 0:
+ return "_private";
+ case 1:
+ return "_global";
+ case 2:
+ return "_constant";
+ case 3:
+ return "_local";
+ case 4:
+ return "_generic";
+ default:
+ return "_as" + std::to_string(AS);
+ }
+}
+
} // namespace
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
@@ -4677,7 +4733,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy = llvm::FunctionType::get(
RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee LoadFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4710,7 +4769,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(1)->getType()));
llvm::FunctionCallee StoreFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4752,7 +4814,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(Arg0->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg1->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg2->getType()));
llvm::FunctionCallee MatMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
@@ -4792,6 +4858,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
(ElTy->isIntegerTy())
? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
: getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ // Function name mangling.
+ Name = Name + getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee BinaryFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
@@ -4821,7 +4889,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
@@ -4849,7 +4919,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarNegFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
@@ -4866,7 +4938,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee InitFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
@@ -4883,7 +4957,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(0)->getType()));
llvm::FunctionCallee LengthFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
new file mode 100644
index 0000000000000..9fec09287f890
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -0,0 +1,174 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+//
+// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
+// Covers three mangling axes:
+// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
+// (B) coop_mat_mulAdd -- different element types (float vs int)
+// (C) coop_mat_load -- different pointer address spaces
+// (D) coop_mat_store -- different pointer address spaces
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
+// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
+// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+// ---------------------------------------------------------------------------
+// Matrix type aliases used across tests
+// ---------------------------------------------------------------------------
+
+// --- float 16x16 (MulAdd group 1) ---
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatF_16x16_A;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatF_16x16_B;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatF_16x16_C;
+
+// --- float 8x32 / 32x8 / 8x8 (MulAdd group 2 -- same elem type, diff dims) ---
+typedef float __attribute__((coop_mat(SCOPE, 8, 32, USE_A))) MatF_8x32_A;
+typedef float __attribute__((coop_mat(SCOPE, 32, 8, USE_B))) MatF_32x8_B;
+typedef float __attribute__((coop_mat(SCOPE, 8, 8, USE_C))) MatF_8x8_C;
+
+// --- int 16x16 (MulAdd group 3 -- different element type to group 1) ---
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatI_16x16_A;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatI_16x16_B;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatI_16x16_C;
+
+// ===========================================================================
+// (A) MulAdd: same element type (float), different matrix dimensions
+// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
+// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
+// ===========================================================================
+
+kernel void test_muladd_f32_16x16(__global float *ptr) {
+ MatF_16x16_A a;
+ MatF_16x16_B b;
+ MatF_16x16_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+kernel void test_muladd_f32_8x32(__global float *ptr) {
+ MatF_8x32_A a;
+ MatF_32x8_B b;
+ MatF_8x8_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 8);
+ b = coop_mat_load(ptr, ROW_MAJOR, 32);
+ c = coop_mat_load(ptr, ROW_MAJOR, 8);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_8x32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x32_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_32x8_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x8_u{{[0-9]+}}
+
+// ===========================================================================
+// (B) MulAdd: different element types -- float 16x16 vs int 16x16
+// Same dimensions, different elem type -> must produce two distinct symbols.
+// ===========================================================================
+
+kernel void test_muladd_i32_16x16(__global int *iptr) {
+ MatI_16x16_A a;
+ MatI_16x16_B b;
+ MatI_16x16_C c, r;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16);
+ b = coop_mat_load(iptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_i32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _i32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+// ===========================================================================
+// (C) Load: different pointer address spaces
+// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
+// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
+// ===========================================================================
+
+kernel void test_load_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_load_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (D) Store: different pointer address spaces
+// Same matrix type, __global vs __local -> two distinct symbols.
+// ===========================================================================
+
+kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (E) Negative: confirm NO cross-contamination between address space variants
+// i.e. the _global load function is NOT called for the _local case.
+// ===========================================================================
+
+kernel void test_no_cross_contamination(__global float *gptr,
+ __local float *lptr) {
+ MatF_16x16_A ag, al;
+ ag = coop_mat_load(gptr, ROW_MAJOR, 16);
+ al = coop_mat_load(lptr, ROW_MAJOR, 16);
+ (void)ag; (void)al;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_no_cross_contamination
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_global_f32_{{.*}}addrspace(3)
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
+
+// Verify distinct declare lines exist (these appear at end of IR).
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index b18f5617338c8..8dd1aab2b2a13 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -26,7 +26,7 @@ typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
// CHECK-LABEL: @{{.*}}test_init_float
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_float(global float *out) {
MatA_t a;
@@ -41,7 +41,7 @@ kernel void test_init_float(global float *out) {
// CHECK-LABEL: @{{.*}}test_init_int
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(i32
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_int(void) {
MatA_int_t m;
@@ -56,7 +56,7 @@ kernel void test_init_int(void) {
// CHECK-LABEL: @{{.*}}test_init_zero
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+// CHECK-SAME: @__spirv_CompositeConstruct_{{.*}}(float {{.*}}0
kernel void test_init_zero(void) {
MatC_t acc;
@@ -71,7 +71,7 @@ kernel void test_init_zero(void) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
kernel void test_length(global unsigned int *out) {
@@ -88,8 +88,8 @@ kernel void test_length(global unsigned int *out) {
// CHECK-LABEL: @{{.*}}test_init_then_length
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: @__spirv_CompositeConstruct
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_init_then_length(global unsigned int *out) {
MatA_t a;
@@ -103,7 +103,7 @@ kernel void test_init_then_length(global unsigned int *out) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length_cc
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_length_cc(global unsigned int *out) {
MatC_t acc;
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index dacb20f022818..3bd2663f235bc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -562,6 +562,21 @@ static bool isNonMangledOCLBuiltin(StringRef Name) {
}
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
+ // Cooperative-matrix builtins emitted by Clang CodeGen carry mangling
+ // suffixes to avoid IR function-signature collisions, e.g.:
+ // "__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global"
+ // Strip everything from the first '_' after "__spirv_CooperativeMatrix"
+ // so the name matches the canonical DemangledNativeBuiltin entry in
+ // SPIRVBuiltins.td (e.g. "__spirv_CooperativeMatrixLoadKHR").
+ static constexpr StringRef CoopMatPrefix = "__spirv_CooperativeMatrix";
+ if (Name.starts_with(CoopMatPrefix)) {
+ size_t SuffixPos = Name.find('_', CoopMatPrefix.size());
+ if (SuffixPos != StringRef::npos)
+ return Name.substr(0, SuffixPos).str();
+ // No mangling suffix present — return the name as-is.
+ return Name.str();
+ }
+
bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
bool IsNonMangledSPIRV = Name.starts_with("__spirv_");
bool IsNonMangledHLSL = Name.starts_with("__hlsl_");
>From 3de062c2159053989f39f84581cae03941536341 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 13:23:45 -0400
Subject: [PATCH 10/17] Change name of extension to cl_khr_cooperative_matrix
---
clang/include/clang/Basic/Builtins.td | 2 +-
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
.../test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../CodeGenOpenCL/coop-mat-init-length-codegen.cl | 2 +-
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 12 ++++++------
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 8 ++++----
clang/test/SemaOpenCL/coop-mat-init-length-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 6 +++---
11 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index cfa9e0801b0f1..e9de3bd6c5795 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,7 +1931,7 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
-// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+// Cooperative Matrix builtins for cl_khr_cooperative_matrix
def CoopMatLoad : Builtin {
let Spellings = ["coop_mat_load"];
let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 89e96dca85a59..8a583ad504c70 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9fec09287f890..9d55bd8c61cc3 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -8,7 +8,7 @@
// (D) coop_mat_store -- different pointer address spaces
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 58b7a882fdc87..ad8df2939269f 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -4,7 +4,7 @@
// SPIR-V intrinsic names.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 8dd1aab2b2a13..dfa763b060e72 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -1,7 +1,7 @@
// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 7181efbe7e2a8..f77fe542fd210 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,6 +1,6 @@
// clang/test/Preprocessor/coop_mat_opencl_ext.cl
//
-// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// Patch 2: cl_khr_cooperative_matrix registration in
// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
//
// Tests: extension macro is predefined when enabled, extension can be
@@ -9,23 +9,23 @@
// ── 2a. Extension macro is predefined when the extension is enabled ─────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
// RUN: | FileCheck %s --check-prefix=EXT
-// EXT: cl_ext_kernel_cooperative_matrix
+// EXT: cl_khr_cooperative_matrix
// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
// RUN: | FileCheck %s --check-prefix=NOEXT
-// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+// NOEXT-NOT: cl_khr_cooperative_matrix
// ── 2c. Enum constants are visible when extension is enabled ─────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index c6230eb94531f..e8fad815806ec 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -6,24 +6,24 @@
//
// ── 3a. Itanium mangling ────────────────────────────────────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch \
// RUN: -ast-dump %t.pch_probe.cl \
// RUN: | FileCheck %s --check-prefix=PCH
//
// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index ab499bc1b4af7..799f7a9e15a35 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -7,7 +7,7 @@
// tested here.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics (sections 1-6 must compile cleanly)
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index f8080448b6df5..0c27978d492a9 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -3,7 +3,7 @@
// Patch 4: Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index cec1f278b0037..bcedd94fdbf17 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -5,7 +5,7 @@
//
// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 2e6700973e611..43adeebd94500 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -8,16 +8,16 @@
// TypeLoc operand slots, sizeof / getTypeInfoImpl.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -ast-dump %s \
// RUN: | FileCheck %s --check-prefix=AST
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
>From a9dbacb8f5eeb58709e16f0edf5cc4b8855aba7c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 14:37:44 -0400
Subject: [PATCH 11/17] Fix test fails
---
clang/lib/Sema/SemaType.cpp | 6 ++++++
clang/test/AST/undocumented-attrs.cpp | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 6e72625f6ac84..a183e7dad91bb 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9002,6 +9002,12 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Scope, Use, /* IsCoopMat */ true);
if (!T.isNull())
CurType = T;
+ } else {
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
}
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index 525466b1efdc0..dff9d3845062a 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,6 +27,7 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
+CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -88,4 +89,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 82
+CHECK-NEXT: Total: 83
>From c882f4d8c0ee6c3805cfc7e9e55b104271ddfcf8 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sun, 9 Aug 2026 00:57:18 -0400
Subject: [PATCH 12/17] Fix failing lldb build and a couple of tests
---
.../test/Misc/pragma-attribute-supported-attributes-list.test | 1 +
lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 3 +++
llvm/test/Verifier/coop-mat-verifier-fix.ll | 4 ++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index ed2022e3b40d9..7ccbfcb6826bc 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -63,6 +63,7 @@
// CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record)
// CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record)
// CHECK-NEXT: Convergent (SubjectMatchRule_function)
+// CHECK-NEXT: CoopMatrixType (SubjectMatchRule_type_alias, SubjectMatchRule_variable, SubjectMatchRule_variable_is_parameter, SubjectMatchRule_field)
// CHECK-NEXT: CoroAwaitElidable (SubjectMatchRule_record)
// CHECK-NEXT: CoroAwaitElidableArgument (SubjectMatchRule_variable_is_parameter)
// CHECK-NEXT: CoroDisableLifetimeBound (SubjectMatchRule_function)
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 5255ec835c0a4..cb4c77d8aaa62 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -4224,6 +4224,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
// Matrix types that we're not sure how to display at the moment.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5095,6 +5096,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type) {
break;
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5268,6 +5270,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
// Matrix types we're not sure how to display yet.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
index c4e206b76b760..7f2eedb657d41 100644
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -22,14 +22,14 @@
declare void @llvm.experimental.noalias.scope.decl(metadata)
define void @test_no_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
; Third arg present and <= 64 bits wide — must also pass
define void @test_valid_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
>From cfd441f53283a9c8fc21b645162fe225ea667457 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 4 Sep 2026 18:58:24 -0400
Subject: [PATCH 13/17] Minor changes to sync with latest upstream changes
---
clang/lib/AST/ASTContext.cpp | 10 ++++------
clang/lib/CodeGen/CGBuiltin.cpp | 2 --
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index c4839bb07c89b..0dcf6a8d2ab68 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4919,9 +4919,8 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
- void *InsertPos = nullptr;
- if (CooperativeMatrixType *MTP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ llvm::FoldingSetInsertToken Token;
+ if (CooperativeMatrixType *MTP = CooperativeMatrixTypes.lookup(ID, Token))
return QualType(MTP, 0);
QualType Canonical;
@@ -4929,15 +4928,14 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
NumRows, NumColumns, Use);
- CooperativeMatrixType *NewIP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ CooperativeMatrixType *NewIP = CooperativeMatrixTypes.lookup(ID, Token);
assert(!NewIP && "Matrix type shouldn't already exist in the map");
(void)NewIP;
}
auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
- CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ CooperativeMatrixTypes.insert(New, Token);
Types.push_back(New);
return QualType(New, 0);
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index a40606ef376d8..dab0eb1e77a13 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2858,8 +2858,6 @@ static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
}
}
-} // namespace
-
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
>From a54cb6c93cdc3b5606f3982711ea92e5f93e0a5e Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 13:55:14 -0400
Subject: [PATCH 14/17] Update CooperativeMatrixType class definition to be
derived directly from Type class, instead of MatrixType class.
---
clang/include/clang/AST/OperationKinds.def | 3 +
clang/include/clang/AST/TypeBase.h | 78 +++++---
clang/include/clang/AST/TypeLoc.h | 105 +++++++----
clang/include/clang/AST/TypeProperties.td | 3 +
clang/include/clang/Basic/TypeNodes.td | 2 +-
clang/include/clang/CIR/Dialect/IR/CIROps.td | 37 ++--
clang/include/clang/Sema/Sema.h | 27 ++-
clang/lib/AST/ASTContext.cpp | 18 +-
clang/lib/AST/Expr.cpp | 1 +
clang/lib/AST/ExprConstant.cpp | 2 +
clang/lib/AST/Type.cpp | 16 --
clang/lib/CIR/CodeGen/CIRGenExpr.cpp | 2 +
clang/lib/CodeGen/CGExpr.cpp | 78 ++++----
clang/lib/CodeGen/CGExprAgg.cpp | 2 +
clang/lib/CodeGen/CGExprComplex.cpp | 1 +
clang/lib/CodeGen/CGExprConstant.cpp | 1 +
clang/lib/CodeGen/CGExprScalar.cpp | 16 +-
clang/lib/CodeGen/CodeGenFunction.h | 23 ++-
clang/lib/CodeGen/CodeGenTBAA.cpp | 5 +-
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +-
clang/lib/CodeGen/QualTypeMapper.h | 1 -
clang/lib/Edit/RewriteObjCFoundationAPI.cpp | 1 +
clang/lib/Sema/SemaCast.cpp | 16 ++
clang/lib/Sema/SemaChecking.cpp | 32 +---
clang/lib/Sema/SemaDecl.cpp | 2 +-
clang/lib/Sema/SemaExpr.cpp | 175 +++++++++++++-----
clang/lib/Sema/SemaTemplateDeduction.cpp | 8 +-
clang/lib/Sema/SemaType.cpp | 145 ++++++++++-----
clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 1 +
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 26 +--
clang/tools/libclang/CIndex.cpp | 2 +-
31 files changed, 540 insertions(+), 295 deletions(-)
diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def
index 8a13ad988403b..d0cda0551f2d5 100644
--- a/clang/include/clang/AST/OperationKinds.def
+++ b/clang/include/clang/AST/OperationKinds.def
@@ -185,6 +185,9 @@ CAST_OPERATION(ToVoid)
/// CK_MatrixCast - A cast between matrix types of the same dimensions.
CAST_OPERATION(MatrixCast)
+/// CK_CoopMatrixCast - A cast between compatible cooperative matrix types.
+CAST_OPERATION(CoopMatrixCast)
+
/// CK_VectorSplat - A conversion from an arithmetic type to a
/// vector of that element type. Fills all elements ("splats") with
/// the source value.
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 560f65a68b8f6..f7d813806eee1 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4494,7 +4494,6 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
- T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4584,65 +4583,81 @@ class ConstantMatrixType final : public MatrixType {
}
};
-/// Represents a cooperative matrix type.
-class CooperativeMatrixType final : public MatrixType {
+/// Represents an opaque OpenCL cooperative matrix type.
+///
+/// Unlike MatrixType, a cooperative matrix is not an ordinary matrix value.
+/// It is an opaque, distributed object whose shape and role are part of its
+/// type identity.
+class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
protected:
friend class ASTContext;
+ /// Element type of the cooperative matrix.
+ QualType ElementType;
+
/// Number of rows and columns.
unsigned NumRows;
unsigned NumColumns;
- /// Scope and use
+ /// Cooperative matrix scope and use.
unsigned Scope;
unsigned Use;
static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
- CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
- unsigned NRows, unsigned NColumns, unsigned Use,
- QualType CanonElementType);
+ CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
+ unsigned NumColumns, unsigned Use,
+ QualType CanonicalType)
+ : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
+ NumRows, NumColumns, Use, CanonicalType) {}
- CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
- unsigned Scope, unsigned NRows, unsigned NColumns,
- unsigned Use, QualType CanonElementType);
+ CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use, QualType CanonicalType)
+ : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
+ Scope(Scope), Use(Use) {}
public:
- /// Returns the number of rows in the matrix.
+ /// Returns the element type.
+ QualType getElementType() const { return ElementType; }
+
+ /// Returns the number of rows.
unsigned getNumRows() const { return NumRows; }
- /// Returns the number of columns in the matrix.
+ /// Returns the number of columns.
unsigned getNumColumns() const { return NumColumns; }
- /// Returns the scope of the matrix.
+ /// Returns the cooperative matrix scope.
unsigned getScope() const { return Scope; }
- /// Returns the use of the matrix.
+ /// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into a vector.
+ /// Returns the number of elements required to embed the matrix into
+ /// a vector representation.
unsigned getNumElementsFlattened() const {
return getNumRows() * getNumColumns();
}
- /// Returns true if \p NumElements is a valid matrix dimension.
+ /// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
}
- /// Return true if \p Scope is valid
+ /// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
+ return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
}
- /// Return true if \p Use is valid
+ /// Returns true if \p Use is a valid cooperative matrix use.
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
+ return Use == 0 || // CLK_COOPERATIVE_MATRIX_A
+ Use == 1 || // CLK_COOPERATIVE_MATRIX_B
+ Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum number of elements per dimension.
+ /// Returns the maximum valid number of elements per dimension.
static constexpr unsigned getMaxElementsPerDimension() {
return MaxElementsPerDimension;
}
@@ -4663,6 +4678,23 @@ class CooperativeMatrixType final : public MatrixType {
ID.AddInteger(TypeClass);
}
+ static bool isValidElementType(QualType ElemTy) {
+ return ElemTy->isSpecificBuiltinType(BuiltinType::Char_S) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UChar) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Short) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UShort) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Int) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UInt) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Long) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::ULong) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Half) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Float) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Double);
+ }
+
+ bool isSugared() const { return false; }
+ QualType desugar() const { return QualType(this, 0); }
+
static bool classof(const Type *T) {
return T->getTypeClass() == CooperativeMatrix;
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 24fcda8f4c7e5..122090f7e6ec5 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,10 +2142,8 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
- Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
- Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2153,50 +2151,26 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
+ /// ^~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
- void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
-
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
- void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
-
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2213,10 +2187,8 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
- setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
- setAttrUseOperand(nullptr);
}
};
@@ -2224,15 +2196,82 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
-class CooperativeMatrixTypeLoc
- : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
- CooperativeMatrixType> {};
-
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
DependentSizedMatrixType> {};
+struct CooperativeMatrixTypeLocInfo {
+ SourceLocation AttrLoc;
+ SourceRange OperandParens;
+ Expr *ScopeOperand;
+ Expr *RowOperand;
+ Expr *ColumnOperand;
+ Expr *UseOperand;
+};
+
+class CooperativeMatrixTypeLoc
+ : public ConcreteTypeLoc<UnqualTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType,
+ CooperativeMatrixTypeLocInfo> {
+public:
+ /// The location of the attribute name, i.e.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^^~~~~~~~~~~~~~~~~~~
+ SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
+ void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+
+ /// The attribute's scope operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
+ /// The attribute's row operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
+ void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
+
+ /// The attribute's column operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
+ void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+
+ /// The attribute's use operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
+ /// The location of the parentheses around the operand, if there is
+ /// an operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^
+ SourceRange getAttrOperandParensRange() const {
+ return getLocalData()->OperandParens;
+ }
+ void setAttrOperandParensRange(SourceRange range) {
+ getLocalData()->OperandParens = range;
+ }
+
+ SourceRange getLocalSourceRange() const {
+ SourceRange range(getAttrNameLoc());
+ range.setEnd(getAttrOperandParensRange().getEnd());
+ return range;
+ }
+
+ void initializeLocal(ASTContext &Context, SourceLocation loc) {
+ setAttrNameLoc(loc);
+ setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
+ setAttrRowOperand(nullptr);
+ setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
+ }
+};
+
// FIXME: location of the '_Complex' keyword.
class ComplexTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
ComplexTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 29ea67a21b299..871c2d8444af2 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -261,6 +261,9 @@ let Class = ConstantMatrixType in {
}
let Class = CooperativeMatrixType in {
+ def : Property<"elementType", QualType> {
+ let Read = [{ node->getElementType() }];
+ }
def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index e6f396245070c..28e4b94c3bea9 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,7 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
-def CooperativeMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<Type>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 70d34884c70a4..24fc00a54caec 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -195,34 +195,35 @@ def CIR_CastKind : CIR_I32EnumAttr<"CastKind", "cast kind", [
I32EnumAttrCase<"ptr_to_bool", 23>,
// CK_ToVoid
// CK_MatrixCast
+ // CK_CoopMatrixCast
// CK_VectorSplat
- I32EnumAttrCase<"integral", 27>,
- I32EnumAttrCase<"int_to_bool", 28>,
- I32EnumAttrCase<"int_to_float", 29>,
+ I32EnumAttrCase<"integral", 28>,
+ I32EnumAttrCase<"int_to_bool", 29>,
+ I32EnumAttrCase<"int_to_float", 30>,
// CK_FloatingToFixedPoint
// CK_FixedPointToFloating
// CK_FixedPointCast
// CK_FixedPointToIntegral
// CK_IntegralToFixedPoint
// CK_FixedPointToBoolean
- I32EnumAttrCase<"float_to_int", 36>,
- I32EnumAttrCase<"float_to_bool", 37>,
- I32EnumAttrCase<"bool_to_int", 38>,
- I32EnumAttrCase<"floating", 39>,
+ I32EnumAttrCase<"float_to_int", 37>,
+ I32EnumAttrCase<"float_to_bool", 38>,
+ I32EnumAttrCase<"bool_to_int", 39>,
+ I32EnumAttrCase<"floating", 40>,
// CK_CPointerToObjCPointerCast
// CK_BlockPointerToObjCPointerCast
// CK_AnyPointerToBlockPointerCast
// CK_ObjCObjectLValueCast
- I32EnumAttrCase<"float_to_complex", 44>,
- I32EnumAttrCase<"float_complex_to_real", 45>,
- I32EnumAttrCase<"float_complex_to_bool", 46>,
- I32EnumAttrCase<"float_complex", 47>,
- I32EnumAttrCase<"float_complex_to_int_complex", 48>,
- I32EnumAttrCase<"int_to_complex", 49>,
- I32EnumAttrCase<"int_complex_to_real", 50>,
- I32EnumAttrCase<"int_complex_to_bool", 51>,
- I32EnumAttrCase<"int_complex", 52>,
- I32EnumAttrCase<"int_complex_to_float_complex", 53>,
+ I32EnumAttrCase<"float_to_complex", 45>,
+ I32EnumAttrCase<"float_complex_to_real", 46>,
+ I32EnumAttrCase<"float_complex_to_bool", 47>,
+ I32EnumAttrCase<"float_complex", 48>,
+ I32EnumAttrCase<"float_complex_to_int_complex", 49>,
+ I32EnumAttrCase<"int_to_complex", 50>,
+ I32EnumAttrCase<"int_complex_to_real", 51>,
+ I32EnumAttrCase<"int_complex_to_bool", 52>,
+ I32EnumAttrCase<"int_complex", 53>,
+ I32EnumAttrCase<"int_complex_to_float_complex", 54>,
// CK_ARCProduceObject
// CK_ARCConsumeObject
// CK_ARCReclaimReturnedObject
@@ -232,7 +233,7 @@ def CIR_CastKind : CIR_I32EnumAttr<"CastKind", "cast kind", [
// CK_CopyAndAutoreleaseBlockObject
// CK_BuiltinFnToFnPtr
// CK_ZeroToOCLOpaqueType
- I32EnumAttrCase<"address_space", 63>,
+ I32EnumAttrCase<"address_space", 64>,
// CK_IntToOCLSampler
// CK_HLSLVectorTruncation
// CK_HLSLArrayRValue
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 587027005c75f..2ef1569932eef 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -7945,6 +7945,14 @@ class Sema final : public SemaBase {
QualType CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
SourceLocation Loc, bool IsCompAssign);
+ /// Type checking for cooperative matrix binary operators.
+ QualType CheckCoopMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+ QualType CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+
/// Are the two types SVE-bitcast-compatible types? I.e. is bitcasting from
/// the first SVE type (e.g. an SVE VLAT) to the second type (e.g. an SVE
/// VLST) allowed?
@@ -7957,6 +7965,11 @@ class Sema final : public SemaBase {
/// do they have the same number of rows and the same number of columns?
bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ /// Are the two types cooperative matrix types and do they have the same
+ /// dimensions i.e. do they have the same number of rows and the same number
+ /// of columns? Also do they have the same scope and use?
+ bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+
bool areVectorTypesSameSize(QualType srcType, QualType destType);
/// Are the two types lax-compatible vector types? That is, given
@@ -7999,6 +8012,13 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
+ // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
+ // We allow casting between cooperative matrixes of the same scope, use, and
+ // same dimensions i.e. when they have the same number of rows and columns.
+ // Returns true if the cast is invalid.
+ bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind);
+
// CheckVectorCast - check type constraints for vectors.
// Since vectors are an extension, there are no C standard reference for this.
// We allow casting between vectors and integer datatypes of the same size.
@@ -15315,8 +15335,11 @@ class Sema final : public SemaBase {
SourceLocation AttrLoc);
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc, Expr *Scope = nullptr,
- Expr *Use = nullptr, bool IsCoopMat = false);
+ SourceLocation AttrLoc);
+
+ QualType BuildCoopMatrixType(QualType T, Expr *Scope, Expr *Use,
+ Expr *NumRows, Expr *NumColumns,
+ SourceLocation AttrLoc);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 0dcf6a8d2ab68..b12ee89e0f82d 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2205,9 +2205,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
case Type::CooperativeMatrix: {
const auto *MT = cast<CooperativeMatrixType>(T);
TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
- // The internal layout of a matrix value is implementation defined.
- // Initially be ABI compatible with arrays with respect to alignment and
- // size.
+ // The internal layout of a cooperative matrix value is implementation
+ // defined. Initially be ABI compatible with arrays with respect to
+ // alignment and size.
Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
Align = ElementInfo.Align;
break;
@@ -10778,10 +10778,10 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatMatrixTypes - Return true if the two specified matrix types are
-/// compatible.
-static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
- const CooperativeMatrixType *RHS) {
+/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
+/// matrix types are compatible.
+static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
return LHS->getElementType() == RHS->getElementType() &&
LHS->getScope() == RHS->getScope() &&
@@ -12305,8 +12305,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
return LHS;
return {};
case Type::CooperativeMatrix:
- if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
- RHSCan->castAs<CooperativeMatrixType>()))
+ if (areCompatCoopMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
return LHS;
return {};
case Type::ObjCObject: {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 6ce0a29aa3bd7..dce415ae8bfd1 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1947,6 +1947,7 @@ bool CastExpr::CastConsistency() const {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
assert(!getType()->isBooleanType() && "unheralded conversion to bool");
goto CheckNoBasePath;
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 3d700e25afe66..85cd6a4e3bd1d 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -20033,6 +20033,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointCast:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLAggregateSplatCast:
llvm_unreachable("invalid cast kind for integral value");
@@ -20966,6 +20967,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLElementwiseCast:
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 4e50e1a57e42c..ee1aa40fd3f15 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -423,20 +423,6 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns) {}
-CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
- nColumns, use, canonType) {}
-
-CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : MatrixType(tc, matrixType, canonType), NumRows(nRows),
- NumColumns(nColumns), Scope(scope), Use(use) {}
-
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -3208,8 +3194,6 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
- if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
- return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index ac7d4693c6000..b77fc3a691150 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -252,6 +252,7 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
case CK_IntegralToFloating:
case CK_LValueBitCast:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_MemberPointerToBoolean:
case CK_NonAtomicToAtomic:
case CK_ObjCObjectLValueCast:
@@ -1626,6 +1627,7 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 897d29a655298..4ae9ab87be034 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7524,58 +7524,68 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy) {
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+llvm::Value *
+CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy) {
+ assert(!Args.empty() &&
+ "Expected at least one argument for cooperative matrix builtin");
+
+ llvm::LLVMContext &Context = CGM.getLLVMContext();
+ llvm::Module &Module = CGM.getModule();
+
+ llvm::Type *ResultLLVMType = ConvertType(ResultTy);
+
+ SmallVector<llvm::Type *, 8> ArgTypes;
+ ArgTypes.reserve(Args.size());
+
+ for (llvm::Value *Arg : Args) {
+ assert(Arg && "Cooperative matrix builtin argument must not be null");
+ ArgTypes.push_back(Arg->getType());
+ }
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+ // Although the frontend builtin is declared as void(...), the LLVM-level
+ // builtin returns the cooperative matrix result directly.
+ llvm::FunctionType *BuiltinType =
+ llvm::FunctionType::get(ResultLLVMType, ArgTypes,
+ /*IsVarArg=*/false);
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
- "coopmat.broadcast");
+ llvm::FunctionCallee Builtin =
+ Module.getOrInsertFunction(BuiltinName, BuiltinType);
+
+ return Builder.CreateCall(Builtin, Args, "coopmat.result");
}
llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
llvm::Value *LHS,
llvm::Value *RHS,
QualType ResultTy) {
- auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
-
switch (Opcode) {
case BO_Add:
- return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_add", {LHS, RHS}, ResultTy);
case BO_Sub:
- return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub(LHS, RHS, "coopmat.isub");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_sub", {LHS, RHS}, ResultTy);
case BO_Mul:
- // mat * mat → element-wise FMul/IMul
- // mat * scalar → handled separately (see scalar path below)
+ // Matrix * scalar.
if (!RHS->getType()->isVectorTy()) {
- // mat * scalar: broadcast scalar then element-wise mul
- llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
- return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_mul", {LHS, RHS},
+ ResultTy);
}
- return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul(LHS, RHS, "coopmat.imul");
+
+ // Matrix * matrix.
+ return EmitCoopMatBuiltinCall("coop_mat_binary_mul", {LHS, RHS}, ResultTy);
case BO_Div:
- if (IsFloat)
- return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned)
- return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
- return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_div", {LHS, RHS}, ResultTy);
default:
- llvm_unreachable("Unsupported cooperative matrix binary op");
+ llvm_unreachable("Unsupported cooperative matrix binary operation");
}
-}
\ No newline at end of file
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatNeg(llvm::Value *Operand,
+ QualType ResultTy) {
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_neg", {Operand}, ResultTy);
+}
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 0c58c4b06b42e..ecb03e473c773 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1114,6 +1114,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
case CK_BuiltinFnToFnPtr:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_IntToOCLSampler:
@@ -1635,6 +1636,7 @@ static bool castPreservesZero(const CastExpr *CE) {
// Language extensions.
case CK_VectorSplat:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_NonAtomicToAtomic:
case CK_AtomicToNonAtomic:
case CK_HLSLVectorTruncation:
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 30d693fb371cb..87877bed010c1 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -618,6 +618,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 48e80910ce577..176847f3ec98b 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1387,6 +1387,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 68d1b4dea16dc..8d00e00324dae 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -3030,7 +3030,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
CGF.EmitIgnoredExpr(E);
return nullptr;
}
- case CK_MatrixCast: {
+ case CK_MatrixCast:
+ case CK_CoopMatrixCast: {
return EmitScalarConversion(Visit(E), E->getType(), DestTy,
CE->getExprLoc());
}
@@ -3696,15 +3697,10 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
- if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
- llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy =
- E->getType()->getAs<CooperativeMatrixType>()->getElementType();
- if (CompTy->isFloatingType())
- return Builder.CreateFNeg(Val, "coopmat.fneg");
- else
- return Builder.CreateNeg(Val, "coopmat.ineg");
- }
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType())
+ return CGF.EmitCoopMatNeg(CGF.EmitScalarExpr(E->getSubExpr()),
+ E->getType());
+
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 7723344bb11d6..a3798fd38341c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5133,15 +5133,6 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
- /// Helper function for EmitCoopMatBinaryOp
- llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy);
-
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
- llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
- llvm::Value *RHS, QualType ResultTy);
-
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
@@ -5481,6 +5472,20 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
SourceLocation loc);
+ /// Helper function to emit coop matrix builtin call.
+ llvm::Value *EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
+ /// EmitCoopMatNeg - Emit the computation of the negate op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
+
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
/// point operation, expressed as the maximum relative error in ulp.
void SetFPAccuracy(llvm::Value *Val, float Accuracy);
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index ecd32fb65e9a1..e368a270f479f 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -331,11 +331,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
+ assert(isa<ConstantMatrixType>(Ty) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
+ if (const auto *CMTy = dyn_cast<CooperativeMatrixType>(Ty))
+ return getTypeInfo(CMTy->getElementType());
+
// Enum types are distinct types. In C++ they have "underlying types",
// however they aren't related for TBAA.
if (const EnumType *ETy = dyn_cast<EnumType>(Ty)) {
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 54d680faaf632..2b380868e22bb 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -109,10 +109,8 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
case Type::CooperativeMatrix: {
- const auto *MT = cast<CooperativeMatrixType>(QT);
- return Builder.getArrayType(convertType(MT->getElementType()),
- MT->getNumRows() * MT->getNumColumns(),
- ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ llvm::reportFatalInternalError(
+ "Cooperative Matrix type not supported in ABI lowering library");
}
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index 76073e980d576..35876f44f3aba 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -47,7 +47,6 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
- const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index e8d4660fd36b2..ecc7805ddd446 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -1079,6 +1079,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
case CK_ZeroToOCLOpaqueType:
case CK_IntToOCLSampler:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
return false;
case CK_BooleanToSignedIntegral:
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 9724c45573d58..81cf070b4422e 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -1607,6 +1607,15 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
return TC_Success;
}
+ if (SrcType->isCooperativeMatrixType() &&
+ DestType->isCooperativeMatrixType()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind)) {
+ SrcExpr = ExprError();
+ return TC_Failed;
+ }
+ return TC_Success;
+ }
+
if (SrcType == Self.Context.AMDGPUFeaturePredicateTy &&
DestType == Self.Context.getLogicalOperationType()) {
SrcExpr = Self.AMDGPU().ExpandAMDGPUPredicateBuiltIn(SrcExpr.get());
@@ -3243,6 +3252,13 @@ void CastOperation::CheckCStyleCast() {
return;
}
+ if (DestType->getAs<CooperativeMatrixType>() ||
+ SrcType->getAs<CooperativeMatrixType>()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind))
+ SrcExpr = ExprError();
+ return;
+ }
+
if (const VectorType *DestVecTy = DestType->getAs<VectorType>()) {
if (Self.CheckAltivecInitFromScalar(OpRange, DestType, SrcType)) {
SrcExpr = ExprError();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9101ba7a27c4d..1b4963f1e1e88 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17333,7 +17333,7 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
ArgError = true;
} else {
ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
- if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
ArgError = true;
}
}
@@ -17453,7 +17453,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
M2Ty->getElementType().getUnqualifiedType())
Diag(Loc, diag::err_coop_matrix_element_type);
- if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
}
@@ -17468,7 +17468,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17590,9 +17590,8 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
- auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
- if (!ConstMType && !CoopMType) {
+ auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
+ if (!MType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17601,23 +17600,12 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- if (ConstMType) {
- QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
- if (CoopMType) {
- QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(), CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
-
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index a91b5a7b150c0..935b24202f939 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14589,7 +14589,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
- if (!VDecl->getType()->isMatrixType()) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 8c023bcade5a6..88745a34b3945 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5146,7 +5146,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
}
// If the base is a matrix type, try to create a new MatrixSubscriptExpr.
- if (base->getType()->isMatrixType()) {
+ if (base->getType()->isMatrixType() ||
+ base->getType()->isCooperativeMatrixType()) {
if (CheckAndReportCommaError(ArgExprs.front()))
return ExprError();
@@ -7952,16 +7953,6 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
- if (srcTy->isCooperativeMatrixType()) {
- const CooperativeMatrixType *matSrcType =
- srcTy->getAs<CooperativeMatrixType>();
- const CooperativeMatrixType *matDestType =
- destTy->getAs<CooperativeMatrixType>();
-
- return matSrcType->getNumRows() == matDestType->getNumRows() &&
- matSrcType->getNumColumns() == matDestType->getNumColumns();
- }
-
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -7969,6 +7960,22 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
+bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
+ QualType destTy) {
+ if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
+ return false;
+
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getScope() == matDestType->getScope() &&
+ matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns() &&
+ matSrcType->getUse() == matDestType->getUse();
+}
+
bool Sema::areVectorTypesSameSize(QualType SrcTy, QualType DestTy) {
assert(DestTy->isVectorType() || SrcTy->isVectorType());
@@ -8074,6 +8081,27 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
return false;
}
+bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind) {
+ if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
+ if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
+ << DestTy << SrcTy << R;
+ }
+ } else if (SrcTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << SrcTy << DestTy << R;
+ } else if (DestTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << DestTy << SrcTy << R;
+ }
+
+ Kind = CK_CoopMatrixCast;
+ return false;
+}
+
bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
CastKind &Kind) {
assert(VectorTy->isVectorType() && "Not a vector type!");
@@ -11306,6 +11334,9 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
ArithConvKind::Arithmetic);
if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
+ if (!IsDiv &&
+ (LHSTy->isCooperativeMatrixType() || RHSTy->isCooperativeMatrixType()))
+ return CheckCoopMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
@@ -11716,7 +11747,7 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -11872,7 +11903,7 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -13891,9 +13922,10 @@ QualType Sema::CheckMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
return InvalidOperands(Loc, LHS, RHS);
}
-QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
- SourceLocation Loc,
- bool IsCompAssign) {
+QualType Sema::CheckCoopMatrixElementwiseOperands(ExprResult &LHS,
+ ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
if (!IsCompAssign) {
LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
if (LHS.isInvalid())
@@ -13903,33 +13935,55 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
- if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
- auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
- auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) &&
- "At least one operand must be a matrix");
- if (LHSMatType && RHSMatType) {
- if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
- return InvalidOperands(Loc, LHS, RHS);
+ // For conversion purposes, we ignore any qualifiers.
+ // For example, "const float" and "float" are equivalent.
+ QualType LHSType = LHS.get()->getType().getUnqualifiedType();
+ QualType RHSType = RHS.get()->getType().getUnqualifiedType();
- if (Context.hasSameType(LHSMatType, RHSMatType))
- return Context.getCommonSugaredType(
- LHS.get()->getType().getUnqualifiedType(),
- RHS.get()->getType().getUnqualifiedType());
+ const CooperativeMatrixType *LHSMatType =
+ LHSType->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *RHSMatType =
+ RHSType->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a cooperative matrix");
- QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
- if (!Context.hasSameType(LHSELTy, RHSELTy))
- return InvalidOperands(Loc, LHS, RHS);
+ if (Context.hasSameType(LHSType, RHSType))
+ return Context.getCommonSugaredType(LHSType, RHSType);
- return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy),
- LHSMatType->getScope(), LHSMatType->getNumRows(),
- RHSMatType->getNumColumns(), LHSMatType->getUse());
- }
+ // Type conversion may change LHS/RHS. Keep copies to the original results, in
+ // case we have to return InvalidOperands.
+ ExprResult OriginalLHS = LHS;
+ ExprResult OriginalRHS = RHS;
+ if (LHSMatType && !RHSMatType) {
+ RHS = tryConvertExprToType(RHS.get(), LHSMatType->getElementType());
+ if (!RHS.isInvalid())
+ return LHSType;
+
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
}
+ if (!LHSMatType && RHSMatType) {
+ LHS = tryConvertExprToType(LHS.get(), RHSMatType->getElementType());
+ if (!LHS.isInvalid())
+ return RHSType;
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
+ }
+
+ return InvalidOperands(Loc, LHS, RHS);
+}
+
+QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -13955,6 +14009,43 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
}
+QualType Sema::CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ return CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+}
+
static bool isLegalBoolVectorBinaryOp(BinaryOperatorKind Opc) {
switch (Opc) {
default:
@@ -15722,7 +15813,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
// Cooperative matrix support
- if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ if (LHSTy->isCooperativeMatrixType() && RHSTy->isCooperativeMatrixType()) {
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
@@ -15730,7 +15821,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
@@ -15760,7 +15851,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
- if (!LHSExpr->getType()->isMatrixType()) {
+ if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
@@ -16430,7 +16521,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
- if (Opc == UO_Minus && Ty->isMatrixType()) {
+ if (Opc == UO_Minus && Ty->isCooperativeMatrixType()) {
return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
{InputExpr});
}
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index c4f2747a95599..347ecd9b88c2b 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2460,9 +2460,11 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
if (!MA)
return TemplateDeductionResult::NonDeducedMismatch;
- // Check that the dimensions are the same
- if (MP->getNumRows() != MA->getNumRows() ||
- MP->getNumColumns() != MA->getNumColumns()) {
+ // Check that the dimensions, scope and use are the same
+ if (MP->getScope() != MA->getScope() ||
+ MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns() ||
+ MP->getUse() != MA->getUse()) {
return TemplateDeductionResult::NonDeducedMismatch;
}
// Perform deduction on element types.
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index a183e7dad91bb..66338e9b98a23 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2495,11 +2495,9 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc, Expr *ScopeExpr,
- Expr *UseExpr, bool IsCoopMat) {
- if (!IsCoopMat)
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc) {
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2580,29 +2578,76 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
- if (IsCoopMat) {
- std::optional<llvm::APSInt> ValueScope =
- ScopeExpr->getIntegerConstantExpr(Context);
- unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
- std::optional<llvm::APSInt> ValueUse =
- UseExpr->getIntegerConstantExpr(Context);
- unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
-
- if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr)
- << ColRange << "matrix scope";
- return QualType();
- }
- if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
- return QualType();
- }
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
- }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
+QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
+ Expr *NumRows, Expr *NumCols, Expr *UseExpr,
+ SourceLocation AttrLoc) {
+ std::optional<llvm::APSInt> ValueRows =
+ NumRows->getIntegerConstantExpr(Context);
+ std::optional<llvm::APSInt> ValueColumns =
+ NumCols->getIntegerConstantExpr(Context);
+
+ auto const RowRange = NumRows->getSourceRange();
+ auto const ColRange = NumCols->getSourceRange();
+
+ // Both are row and column expressions are invalid.
+ if (!ValueRows && !ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange << ColRange;
+ return QualType();
+ }
+
+ // Only the row expression is invalid.
+ if (!ValueRows) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange;
+ return QualType();
+ }
+
+ // Only the column expression is invalid.
+ if (!ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << ColRange;
+ return QualType();
+ }
+
+ // Check the matrix dimensions.
+ unsigned MatrixRows = static_cast<unsigned>(ValueRows->getZExtValue());
+ unsigned MatrixColumns = static_cast<unsigned>(ValueColumns->getZExtValue());
+ if (MatrixRows == 0 && MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size)
+ << "matrix" << RowRange << ColRange;
+ return QualType();
+ }
+ if (MatrixRows == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << RowRange;
+ return QualType();
+ }
+ if (MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << ColRange;
+ return QualType();
+ }
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+}
+
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
if ((T->isArrayType() && !getLangOpts().allowArrayReturnTypes()) ||
T->isFunctionType()) {
@@ -5977,7 +6022,7 @@ static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
}
}
- llvm_unreachable("no matrix_type attribute found at the expected location!");
+ llvm_unreachable("no coop_mat attribute found at the expected location!");
}
namespace {
@@ -8981,34 +9026,40 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S, bool IsCoopMat = false) {
- if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
+ Sema &S) {
+ if (!S.getLangOpts().MatrixTypes) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- unsigned int NumAttrs = IsCoopMat ? 4 : 2;
- if (Attr.getNumArgs() != NumAttrs) {
+ if (Attr.getNumArgs() != 2) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << NumAttrs;
+ << Attr << 2;
return;
}
- if (IsCoopMat) {
- Expr *Scope = Attr.getArgAsExpr(0);
- Expr *RowsExpr = Attr.getArgAsExpr(1);
- Expr *ColsExpr = Attr.getArgAsExpr(2);
- Expr *Use = Attr.getArgAsExpr(3);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
- Scope, Use, /* IsCoopMat */ true);
- if (!T.isNull())
- CurType = T;
- } else {
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
+}
+
+/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
+ Sema &S) {
+ if (Attr.getNumArgs() != 4) {
+ S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+ << Attr << 4;
+ return;
}
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildCoopMatrixType(CurType, Scope, RowsExpr, ColsExpr, Use,
+ Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9281,7 +9332,7 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
break;
case ParsedAttr::AT_CoopMatrixType:
- HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ HandleCoopMatrixTypeAttr(type, attr, state.getSema());
attr.setUsedAsTypeAttr();
break;
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index 6127328cefe23..54b8c1210596d 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -510,6 +510,7 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
// Various C++ casts that are not handled yet.
case CK_ToUnion:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_VectorSplat:
case CK_HLSLElementwiseCast:
case CK_HLSLAggregateSplatCast:
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 43adeebd94500..f4c78b965d24a 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -30,7 +30,7 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
// ---------------------------------------------------------------------------
-// 1. Basic type construction — four use roles, float element type.
+// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
@@ -43,7 +43,7 @@ typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
// AST: TypedefDecl {{.*}} MatA_int8x8
// ---------------------------------------------------------------------------
-// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// Type printer — VarDecls carry the coop_mat attribute in their type string.
// ---------------------------------------------------------------------------
void test_type_spelling(void) {
MatA_float16x16 a;
@@ -58,17 +58,7 @@ void test_type_spelling(void) {
// AST: VarDecl {{.*}} d {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
-// float(4B)*16*16 = 1024 B = 8192 bits
-// int(4B)*8*8 = 256 B = 2048 bits
-// ---------------------------------------------------------------------------
-void test_sizeof(void) {
- _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
- _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
-}
-
-// ---------------------------------------------------------------------------
-// 4. Parameter / return type — type preserved through function boundary.
+// Parameter / return type — type preserved through function boundary.
// ---------------------------------------------------------------------------
MatA_float16x16 test_param_return(MatA_float16x16 in) {
return in;
@@ -78,7 +68,7 @@ MatA_float16x16 test_param_return(MatA_float16x16 in) {
// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 5. TypeLoc operand traversal — all four operand slots populated.
+// TypeLoc operand traversal — all four operand slots populated.
// ---------------------------------------------------------------------------
void test_typeloc_operands(void) {
float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
@@ -88,7 +78,7 @@ void test_typeloc_operands(void) {
// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// RecursiveASTVisitor — element type (half) reachable through the node.
// ---------------------------------------------------------------------------
void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
(void)x;
@@ -97,8 +87,8 @@ void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))
// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
-// same canonical type; taking a pointer across them compiles cleanly.
+// mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
@@ -109,6 +99,6 @@ void test_merge_types(void) {
}
// ---------------------------------------------------------------------------
-// 8. PCH serialisation round-trip.
+// PCH serialisation round-trip.
// ---------------------------------------------------------------------------
// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 3e7320deacdbb..9b2f6edc985ea 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1909,7 +1909,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
-DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, Type)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
>From 6d9e9f8db2b4b504dfddd0260a0688be96dd40bb Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 14:31:27 -0400
Subject: [PATCH 15/17] iMinor issues with previous commit
---
clang/include/clang/AST/TypeLoc.h | 2 +-
clang/lib/CodeGen/CGExpr.cpp | 2 +-
clang/lib/Sema/SemaChecking.cpp | 3 +--
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 122090f7e6ec5..b7432d03a92dd 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2217,7 +2217,7 @@ class CooperativeMatrixTypeLoc
public:
/// The location of the attribute name, i.e.
/// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^^~~~~~~~~~~~~~~~~~~
+ /// ^~~~~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 4ae9ab87be034..9d4debfcc3275 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -6236,6 +6236,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
@@ -7531,7 +7532,6 @@ CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
assert(!Args.empty() &&
"Expected at least one argument for cooperative matrix builtin");
- llvm::LLVMContext &Context = CGM.getLLVMContext();
llvm::Module &Module = CGM.getModule();
llvm::Type *ResultLLVMType = ConvertType(ResultTy);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 1b4963f1e1e88..967a3c45d49f1 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17601,8 +17601,7 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
>From 8900675fc5bd528c04096cf89c121d40456e6272 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 15:35:07 -0400
Subject: [PATCH 16/17] Add regression test for change in SPIRVUtils.cpp
---
.../SPIRV/coop-matrix-builtin-demangle.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
diff --git a/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
new file mode 100644
index 0000000000000..e99d2199b7131
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=spirv64-unknown-unknown \
+; RUN: -spirv-ext=+SPV_KHR_cooperative_matrix \
+; RUN: -filetype=asm < %s | FileCheck %s
+
+target triple = "spirv64-unknown-unknown"
+
+%coopmat.f32.sc3.16x16.u0 =
+ type target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+
+; ---------------------------------------------------------------------------
+; Suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_mangled_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_mangled_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
+; ---------------------------------------------------------------------------
+; No-suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_canonical_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_canonical_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
>From 044d3cd77bb4ec64ca7448d51ecff5bd147e97b6 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 17:00:22 -0400
Subject: [PATCH 17/17] Address review comments
---
clang/include/clang/AST/TypeBase.h | 19 ++------
clang/include/clang/AST/TypeProperties.td | 6 +--
.../include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/lib/AST/ASTStructuralEquivalence.cpp | 2 +-
clang/lib/Headers/opencl-c-base.h | 47 ++++++++++---------
5 files changed, 33 insertions(+), 43 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index f7d813806eee1..78482f7918cd9 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4608,13 +4608,8 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
- : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
- NumRows, NumColumns, Use, CanonicalType) {}
-
- CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
- unsigned Scope, unsigned NumRows, unsigned NumColumns,
- unsigned Use, QualType CanonicalType)
- : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ : Type(Type::CooperativeMatrix, CanonicalType,
+ ElementType->getDependence()),
ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
Scope(Scope), Use(Use) {}
@@ -4634,12 +4629,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into
- /// a vector representation.
- unsigned getNumElementsFlattened() const {
- return getNumRows() * getNumColumns();
- }
-
/// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
@@ -4663,12 +4652,12 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
}
void Profile(llvm::FoldingSetNodeID &ID) {
- Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
}
static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
- unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
unsigned Use, TypeClass TypeClass) {
ID.AddPointer(ElementType.getAsOpaquePtr());
ID.AddInteger(NumRows);
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 871c2d8444af2..f7b2c0b6f79bf 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -264,15 +264,15 @@ let Class = CooperativeMatrixType in {
def : Property<"elementType", QualType> {
let Read = [{ node->getElementType() }];
}
- def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
let Read = [{ node->getNumColumns() }];
}
- def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+ def : Property<"use", UInt32> { let Read = [{ node->getUse() }]; }
def : Creator<[{
- return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ return ctx.getCooperativeMatrixType(elementType, scope, numRows, numColumns, use);
}]>;
}
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 8a583ad504c70..56ce32cf3e7c5 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 8ec1100f9f4d8..fd4f55e43f665 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1155,7 +1155,7 @@ bool ASTStructuralEquivalence::isEquivalent(
const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
// The element types must be structurally equivalent and the number of rows
- // and columns must match.
+ // and columns and scope and use must match.
if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
Mat2->getElementType()) ||
Mat1->getScope() != Mat2->getScope() ||
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 7b3d239299c6e..83bb35f1d886a 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,6 +632,30 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
+
#ifdef cl_intel_device_side_avc_motion_estimation
#define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0
@@ -771,27 +795,4 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
-typedef enum coop_matrix_operands_t {
- CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
- CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
-} coop_matrix_operands_t;
-
-typedef enum coop_matrix_use_t {
- CLK_COOPERATIVE_MATRIX_A = 0,
- CLK_COOPERATIVE_MATRIX_B = 1,
- CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
-} coop_matrix_use_t;
-
-typedef enum coop_matrix_layout_t {
- CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
- CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
-} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
More information about the cfe-commits
mailing list