[clang] [lldb] [llvm] [OpenCL][Clang] Add support for cooperative matrix extension (PR #221328)
via cfe-commits
cfe-commits at lists.llvm.org
Sat Oct 17 10:39:16 PDT 2026
https://github.com/asudarsa-qti updated https://github.com/llvm/llvm-project/pull/221328
>From 01b9d8253909c4393b93ba0dc853f40b7d3d9b7c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:14:42 -0400
Subject: [PATCH 01/48] [OpenCL][CoopMat] Patch 1 - Introduce Cooperative
matrix type
---
clang/include/clang/AST/ASTContext.h | 10 +++
clang/include/clang/AST/RecursiveASTVisitor.h | 11 +++
clang/include/clang/AST/TypeBase.h | 90 +++++++++++++++++++
clang/include/clang/AST/TypeLoc.h | 34 ++++++-
clang/include/clang/AST/TypeProperties.td | 13 +++
clang/include/clang/Basic/TypeNodes.td | 1 +
.../clang/Serialization/TypeBitCodes.def | 1 +
clang/lib/AST/ASTContext.cpp | 82 +++++++++++++++++
clang/lib/AST/Type.cpp | 38 ++++++++
clang/lib/AST/TypePrinter.cpp | 15 ++++
10 files changed, 294 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 04cc2da25cecbb..87139685e9f3a9 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -270,6 +270,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
mutable llvm::ContextualFoldingSet<DependentVectorType, ASTContext &>
DependentVectorTypes;
mutable llvm::FoldingSet<ConstantMatrixType> MatrixTypes;
+ mutable llvm::FoldingSet<CooperativeMatrixType> CooperativeMatrixTypes;
mutable llvm::ContextualFoldingSet<DependentSizedMatrixType, ASTContext &>
DependentSizedMatrixTypes;
mutable llvm::FoldingSet<FunctionNoProtoType> FunctionNoProtoTypes;
@@ -1923,6 +1924,15 @@ class ASTContext : public RefCountedBase<ASTContext> {
QualType ElementType, unsigned NumRows, unsigned NumColumns,
std::optional<MatrixType::LayoutKind> Layout = std::nullopt) const;
+ /// Return the unique reference to the cooperative matrix type of the
+ /// specified element type and size.
+ ///
+ /// \pre \p ElementType must be a valid matrix element type (see
+ /// MatrixType::isValidElementType).
+ QualType getCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use) const;
+
/// Return the unique reference to the matrix type of the specified element
/// type and size
QualType getDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index f144c890f6ee65..791536d03f8f42 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -1089,6 +1089,9 @@ DEF_TRAVERSE_TYPE(ExtVectorType, { TRY_TO(TraverseType(T->getElementType())); })
DEF_TRAVERSE_TYPE(ConstantMatrixType,
{ TRY_TO(TraverseType(T->getElementType())); })
+DEF_TRAVERSE_TYPE(CooperativeMatrixType,
+ { TRY_TO(TraverseType(T->getElementType())); })
+
DEF_TRAVERSE_TYPE(DependentSizedMatrixType, {
if (T->getRowExpr())
TRY_TO(TraverseStmt(T->getRowExpr()));
@@ -1409,6 +1412,14 @@ DEF_TRAVERSE_TYPELOC(ConstantMatrixType, {
TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
})
+DEF_TRAVERSE_TYPELOC(CooperativeMatrixType, {
+ TRY_TO(TraverseStmt(TL.getAttrScopeOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrUseOperand()));
+ TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
+})
+
DEF_TRAVERSE_TYPELOC(DependentSizedMatrixType, {
TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 6373794bb69f54..9b1c9ef646f1f5 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -2708,6 +2708,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
bool isSubscriptableVectorType() const;
bool isMatrixType() const; // Matrix type.
bool isConstantMatrixType() const; // Constant matrix type.
+ bool isCooperativeMatrixType() const; // Cooperative matrix type.
bool isOverflowBehaviorType() const; // Overflow behavior type.
bool isDependentAddressSpaceType() const; // value-dependent address space qualifier
bool isObjCObjectPointerType() const; // pointer to ObjC object
@@ -4482,6 +4483,7 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
+ T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4577,6 +4579,90 @@ class ConstantMatrixType final : public MatrixType {
}
};
+/// Represents a cooperative matrix type.
+class CooperativeMatrixType final : public MatrixType {
+protected:
+ friend class ASTContext;
+
+ /// Number of rows and columns.
+ unsigned NumRows;
+ unsigned NumColumns;
+
+ /// Scope and use
+ unsigned Scope;
+ unsigned Use;
+
+ static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
+
+ CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
+ unsigned NRows, unsigned NColumns, unsigned Use,
+ QualType CanonElementType);
+
+ CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
+ unsigned Scope, unsigned NRows, unsigned NColumns,
+ unsigned Use, QualType CanonElementType);
+
+public:
+ /// Returns the number of rows in the matrix.
+ unsigned getNumRows() const { return NumRows; }
+
+ /// Returns the number of columns in the matrix.
+ unsigned getNumColumns() const { return NumColumns; }
+
+ /// Returns the scope of the matrix.
+ unsigned getScope() const { return Scope; }
+
+ /// Returns the use of the matrix.
+ unsigned getUse() const { return Use; }
+
+ /// Returns the number of elements required to embed the matrix into a vector.
+ unsigned getNumElementsFlattened() const {
+ return getNumRows() * getNumColumns();
+ }
+
+ /// Returns true if \p NumElements is a valid matrix dimension.
+ static constexpr bool isDimensionValid(size_t NumElements) {
+ return NumElements > 0 && NumElements <= MaxElementsPerDimension;
+ }
+
+ /// Return true if \p Scope is valid
+ static constexpr bool isScopeValid(size_t Scope) {
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ }
+
+ /// Return true if \p Use is valid
+ static constexpr bool isUseValid(size_t Use) {
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ }
+
+ /// Returns the maximum number of elements per dimension.
+ static constexpr unsigned getMaxElementsPerDimension() {
+ return MaxElementsPerDimension;
+ }
+
+ void Profile(llvm::FoldingSetNodeID &ID) {
+ Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ getUse(), getTypeClass());
+ }
+
+ static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
+ unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Use, TypeClass TypeClass) {
+ ID.AddPointer(ElementType.getAsOpaquePtr());
+ ID.AddInteger(NumRows);
+ ID.AddInteger(NumColumns);
+ ID.AddInteger(Scope);
+ ID.AddInteger(Use);
+ ID.AddInteger(TypeClass);
+ }
+
+ static bool classof(const Type *T) {
+ return T->getTypeClass() == CooperativeMatrix;
+ }
+};
+
/// Represents a matrix type where the type and the number of rows and columns
/// is dependent on a template.
class DependentSizedMatrixType final : public MatrixType {
@@ -8851,6 +8937,10 @@ inline bool Type::isConstantMatrixType() const {
return isa<ConstantMatrixType>(CanonicalType);
}
+inline bool Type::isCooperativeMatrixType() const {
+ return isa<CooperativeMatrixType>(CanonicalType);
+}
+
inline bool Type::isOverflowBehaviorType() const {
return isa<OverflowBehaviorType>(CanonicalType);
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index b3cb4d0f336295..24fcda8f4c7e58 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,8 +2142,10 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
+ Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
+ Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2151,26 +2153,50 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^~~~~~~~~~~~~~~~~
+ /// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2187,8 +2213,10 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
}
};
@@ -2196,6 +2224,10 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
+class CooperativeMatrixTypeLoc
+ : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType> {};
+
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 1a2a3fda0dcff0..88d84d6d3234f5 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -263,6 +263,19 @@ let Class = ConstantMatrixType in {
}]>;
}
+let Class = CooperativeMatrixType in {
+ def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
+ def : Property<"numColumns", UInt32> {
+ let Read = [{ node->getNumColumns() }];
+ }
+ def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+
+ def : Creator<[{
+ return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ }]>;
+}
+
let Class = DependentSizedMatrixType in {
def : Property<"rows", ExprRef> {
let Read = [{ node->getRowExpr() }];
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 700a73f6696903..e6f396245070c6 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,6 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<MatrixType>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index 9f1da65e0f9401..c9ebac63390f5e 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -70,5 +70,6 @@ TYPE_BIT_CODE(HLSLInlineSpirv, HLSL_INLINE_SPIRV, 60)
TYPE_BIT_CODE(PredefinedSugar, PREDEFINED_SUGAR, 61)
TYPE_BIT_CODE(SubstBuiltinTemplatePack, SUBST_BUILTIN_TEMPLATE_PACK, 62)
TYPE_BIT_CODE(OverflowBehavior, OVERFLOWBEHAVIOR, 63)
+TYPE_BIT_CODE(CooperativeMatrix, COOPERATIVE_MATRIX, 64)
#undef TYPE_BIT_CODE
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index ed22919b7a70bb..93fc3b3da69ff3 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2219,6 +2219,17 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
break;
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(T);
+ TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
+ // The internal layout of a matrix value is implementation defined.
+ // Initially be ABI compatible with arrays with respect to alignment and
+ // size.
+ Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
+ Align = ElementInfo.Align;
+ break;
+ }
+
case Type::Builtin:
switch (cast<BuiltinType>(T)->getKind()) {
default: llvm_unreachable("Unknown builtin type!");
@@ -3561,6 +3572,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
case Type::Pipe:
case Type::BitInt:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
OS << "?";
return;
@@ -4396,6 +4408,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
case Type::ExtVector:
case Type::DependentSizedExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::ObjCObject:
@@ -4943,6 +4956,45 @@ QualType ASTContext::getConstantMatrixType(
return QualType(New, 0);
}
+QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
+ unsigned Scope, unsigned NumRows,
+ unsigned NumColumns,
+ unsigned Use) const {
+ llvm::FoldingSetNodeID ID;
+ CooperativeMatrixType::Profile(ID, ElementTy, Scope, NumRows, NumColumns, Use,
+ Type::CooperativeMatrix);
+
+ assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
+ "need a valid element type");
+ assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
+ CooperativeMatrixType::isDimensionValid(NumColumns) &&
+ "need valid matrix dimensions");
+ assert(CooperativeMatrixType::isScopeValid(Scope) &&
+ "need valid matrix scope");
+ assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
+ void *InsertPos = nullptr;
+ if (CooperativeMatrixType *MTP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ return QualType(MTP, 0);
+
+ QualType Canonical;
+ if (!ElementTy.isCanonical()) {
+ Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
+ NumRows, NumColumns, Use);
+
+ CooperativeMatrixType *NewIP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ assert(!NewIP && "Matrix type shouldn't already exist in the map");
+ (void)NewIP;
+ }
+
+ auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
+ ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
+ CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ Types.push_back(New);
+ return QualType(New, 0);
+}
+
QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
Expr *RowExpr,
Expr *ColumnExpr,
@@ -9811,6 +9863,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
return;
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
if (NotEncodedT)
*NotEncodedT = T;
return;
@@ -10779,6 +10832,18 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
+/// areCompatMatrixTypes - Return true if the two specified matrix types are
+/// compatible.
+static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
+ assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
+ return LHS->getElementType() == RHS->getElementType() &&
+ LHS->getScope() == RHS->getScope() &&
+ LHS->getNumRows() == RHS->getNumRows() &&
+ LHS->getNumColumns() == RHS->getNumColumns() &&
+ LHS->getUse() == RHS->getUse();
+}
+
bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
QualType SecondVec) {
assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
@@ -12293,6 +12358,11 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
RHSCan->castAs<ConstantMatrixType>()))
return LHS;
return {};
+ case Type::CooperativeMatrix:
+ if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
+ return LHS;
+ return {};
case Type::ObjCObject: {
// Check if the types are assignment compatible.
// FIXME: This should be type compatibility, e.g. whether
@@ -14698,6 +14768,17 @@ static QualType getCommonNonSugarTypeNode(const ASTContext &Ctx, const Type *X,
return Ctx.getConstantMatrixType(getCommonElementType(Ctx, MX, MY),
MX->getNumRows(), MX->getNumColumns());
}
+ case Type::CooperativeMatrix: {
+ const auto *MX = cast<CooperativeMatrixType>(X),
+ *MY = cast<CooperativeMatrixType>(Y);
+ assert(MX->getScope() == MY->getScope());
+ assert(MX->getNumRows() == MY->getNumRows());
+ assert(MX->getNumColumns() == MY->getNumColumns());
+ assert(MX->getUse() == MY->getUse());
+ return Ctx.getCooperativeMatrixType(getCommonElementType(Ctx, MX, MY),
+ MX->getScope(), MX->getNumRows(),
+ MX->getNumColumns(), MX->getUse());
+ }
case Type::DependentSizedMatrix: {
const auto *MX = cast<DependentSizedMatrixType>(X),
*MY = cast<DependentSizedMatrixType>(Y);
@@ -14852,6 +14933,7 @@ static QualType getCommonSugarTypeNode(const ASTContext &Ctx, const Type *X,
CANONICAL_TYPE(ConstantArray)
CANONICAL_TYPE(ArrayParameter)
CANONICAL_TYPE(ConstantMatrix)
+ CANONICAL_TYPE(CooperativeMatrix)
CANONICAL_TYPE(Enum)
CANONICAL_TYPE(ExtVector)
CANONICAL_TYPE(FunctionNoProto)
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 4c5f2f60a90005..020564f3edfb79 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -510,6 +510,20 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns), Layout(Layout) {}
+CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
+ nColumns, use, canonType) {}
+
+CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : MatrixType(tc, matrixType, canonType), NumRows(nRows),
+ NumColumns(nColumns), Scope(scope), Use(use) {}
+
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -1286,6 +1300,18 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
T->getNumColumns(), T->getLayout());
}
+ QualType VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ QualType elementType = recurse(T->getElementType());
+ if (elementType.isNull())
+ return {};
+ if (elementType.getAsOpaquePtr() == T->getElementType().getAsOpaquePtr())
+ return QualType(T, 0);
+
+ return Ctx.getCooperativeMatrixType(elementType, T->getScope(),
+ T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ }
+
QualType VisitOverflowBehaviorType(const OverflowBehaviorType *T) {
QualType UnderlyingType = recurse(T->getUnderlyingType());
if (UnderlyingType.isNull())
@@ -2193,6 +2219,10 @@ class GetContainedDeducedTypeVisitor
return Visit(T->getElementType());
}
+ Type *VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+ }
+
Type *VisitFunctionProtoType(const FunctionProtoType *T) {
if (Syntactic && T->hasTrailingReturn())
return const_cast<FunctionProtoType *>(T);
@@ -3274,6 +3304,8 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
+ if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
+ return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
@@ -5166,6 +5198,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
return Cache::get(cast<VectorType>(T)->getElementType());
case Type::ConstantMatrix:
return Cache::get(cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return Cache::get(cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return Cache::get(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5268,6 +5302,9 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
case Type::ConstantMatrix:
return computeTypeLinkageInfo(
cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return computeTypeLinkageInfo(
+ cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return computeTypeLinkageInfo(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5472,6 +5509,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::FunctionProto:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 13929f6fe08005..34380e3c8965f8 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -280,6 +280,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -917,6 +918,20 @@ void TypePrinter::printConstantMatrixAfter(const ConstantMatrixType *T,
printAfter(T->getElementType(), OS);
}
+void TypePrinter::printCooperativeMatrixBefore(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printBefore(T->getElementType(), OS);
+ OS << " __attribute__((coop_mat(";
+ OS << T->getScope() << ", " << T->getNumRows() << ", ";
+ OS << T->getNumColumns() << ", " << T->getUse();
+ OS << ")))";
+}
+
+void TypePrinter::printCooperativeMatrixAfter(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printAfter(T->getElementType(), OS);
+}
+
void TypePrinter::printDependentSizedMatrixBefore(
const DependentSizedMatrixType *T, raw_ostream &OS) {
if (Policy.UseHLSLTypes) {
>From beef7815124560ae828110a6ee589eb760b55c11 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:15:43 -0400
Subject: [PATCH 02/48] Patch 2 - Define extension macro
---
.../include/clang/Basic/OpenCLExtensions.def | 1 +
clang/lib/Headers/opencl-c-base.h | 23 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index f8af9b843f95f4..89e96dca85a59d 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,6 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
+OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 251894ac5a2630..7b3d239299c6e7 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -771,4 +771,27 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From c894f74275f8475dbc55fe683f2a1525c377e6eb Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:16:39 -0400
Subject: [PATCH 03/48] Patch 3 - Add support for serialization, round-trip and
mangling
---
clang/lib/AST/ASTImporter.cpp | 11 +++++++++
clang/lib/AST/ASTStructuralEquivalence.cpp | 15 +++++++++++++
clang/lib/AST/ExprConstant.cpp | 1 +
clang/lib/AST/ItaniumMangle.cpp | 26 ++++++++++++++++++++++
clang/lib/AST/MicrosoftMangle.cpp | 21 +++++++++++++++++
clang/lib/Serialization/ASTReader.cpp | 10 +++++++++
clang/lib/Serialization/ASTWriter.cpp | 12 ++++++++++
7 files changed, 96 insertions(+)
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 4c296b6829b1b0..1bf29a062287bd 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -2116,6 +2116,17 @@ ExpectedType clang::ASTNodeImporter::VisitConstantMatrixType(
*ToElementTypeOrErr, T->getNumRows(), T->getNumColumns(), T->getLayout());
}
+ExpectedType clang::ASTNodeImporter::VisitCooperativeMatrixType(
+ const clang::CooperativeMatrixType *T) {
+ ExpectedType ToElementTypeOrErr = import(T->getElementType());
+ if (!ToElementTypeOrErr)
+ return ToElementTypeOrErr.takeError();
+
+ return Importer.getToContext().getCooperativeMatrixType(
+ *ToElementTypeOrErr, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+}
+
ExpectedType clang::ASTNodeImporter::VisitDependentAddressSpaceType(
const clang::DependentAddressSpaceType *T) {
Error Err = Error::success();
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 029e33ae0db760..8ec1100f9f4d89 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1151,6 +1151,21 @@ bool ASTStructuralEquivalence::isEquivalent(
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
+ const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
+ // The element types must be structurally equivalent and the number of rows
+ // and columns must match.
+ if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
+ Mat2->getElementType()) ||
+ Mat1->getScope() != Mat2->getScope() ||
+ Mat1->getNumRows() != Mat2->getNumRows() ||
+ Mat1->getNumColumns() != Mat2->getNumColumns() ||
+ Mat1->getUse() != Mat2->getUse())
+ return false;
+ break;
+ }
+
case Type::FunctionProto: {
const auto *Proto1 = cast<FunctionProtoType>(T1);
const auto *Proto2 = cast<FunctionProtoType>(T2);
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 6d390032e2c717..554839160228fc 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -16583,6 +16583,7 @@ GCCTypeClass EvaluateBuiltinClassifyType(QualType T,
case Type::BlockPointer:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::ObjCObject:
case Type::ObjCInterface:
case Type::ObjCObjectPointer:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index ddde9d48606df3..b6b832dc042d01 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2550,6 +2550,7 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -4526,6 +4527,31 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
Out << "E";
}
+void CXXNameMangler::mangleType(const CooperativeMatrixType *T) {
+ // Mangle cooperative matrix types as a vendor extended type:
+ // u<Len>coop_matI<Scope><Rows><Columns><Use><element type>E
+
+ mangleVendorType("coop_mat");
+
+ Out << "I";
+ auto &ASTCtx = getASTContext();
+ unsigned BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
+ llvm::APSInt Scope(BitWidth);
+ Scope = T->getScope();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Scope);
+ llvm::APSInt Rows(BitWidth);
+ Rows = T->getNumRows();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Rows);
+ llvm::APSInt Columns(BitWidth);
+ Columns = T->getNumColumns();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Columns);
+ llvm::APSInt Use(BitWidth);
+ Use = T->getUse();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Use);
+ mangleType(T->getElementType());
+ Out << "E";
+}
+
void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
// Mangle matrix types as a vendor extended type:
// u<Len>matrix_typeI<row expr><column expr><element type>E
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 74edee531e48a0..612979b002b596 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3810,6 +3810,27 @@ void MicrosoftCXXNameMangler::mangleType(const ConstantMatrixType *T,
mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
}
+void MicrosoftCXXNameMangler::mangleType(const CooperativeMatrixType *T,
+ Qualifiers quals, SourceRange Range) {
+ QualType EltTy = T->getElementType();
+
+ llvm::SmallString<64> TemplateMangling;
+ llvm::raw_svector_ostream Stream(TemplateMangling);
+ MicrosoftCXXNameMangler Extra(Context, Stream);
+
+ Stream << "?$";
+
+ Extra.mangleSourceName("__coop_mat");
+ Extra.mangleType(EltTy, Range, QMM_Escape);
+
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getScope()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumRows()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumColumns()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getUse()));
+
+ mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
+}
+
void MicrosoftCXXNameMangler::mangleType(const DependentSizedMatrixType *T,
Qualifiers quals, SourceRange Range) {
Error(Range.getBegin(), "dependent-sized matrix type") << Range;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index e6f6be496e5e58..9d8fea3651cdf4 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7693,6 +7693,16 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ TL.setAttrNameLoc(readSourceLocation());
+ TL.setAttrOperandParensRange(readSourceRange());
+ TL.setAttrScopeOperand(Reader.readExpr());
+ TL.setAttrRowOperand(Reader.readExpr());
+ TL.setAttrColumnOperand(Reader.readExpr());
+ TL.setAttrUseOperand(Reader.readExpr());
+}
+
void TypeLocReader::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 0afba1c84fd627..42d6f33e0711e1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -457,6 +457,18 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ addSourceLocation(TL.getAttrNameLoc());
+ SourceRange range = TL.getAttrOperandParensRange();
+ addSourceLocation(range.getBegin());
+ addSourceLocation(range.getEnd());
+ Record.AddStmt(TL.getAttrScopeOperand());
+ Record.AddStmt(TL.getAttrRowOperand());
+ Record.AddStmt(TL.getAttrColumnOperand());
+ Record.AddStmt(TL.getAttrUseOperand());
+}
+
void TypeLocWriter::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
>From ab67434e8eda1ca159fd41173568d68fffd59e42 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:21 -0400
Subject: [PATCH 04/48] Patch 4 - Add support for semantic analysis and
checking
---
clang/include/clang/Basic/Attr.td | 8 +
.../clang/Basic/DiagnosticSemaKinds.td | 54 +++
clang/include/clang/Sema/Sema.h | 34 +-
clang/lib/Sema/SemaChecking.cpp | 310 +++++++++++++++++-
clang/lib/Sema/SemaDecl.cpp | 32 ++
clang/lib/Sema/SemaExpr.cpp | 131 +++++++-
clang/lib/Sema/SemaLookup.cpp | 1 +
clang/lib/Sema/SemaTemplate.cpp | 5 +
clang/lib/Sema/SemaTemplateDeduction.cpp | 31 ++
clang/lib/Sema/SemaType.cpp | 78 ++++-
clang/lib/Sema/TreeTransform.h | 42 +++
11 files changed, 704 insertions(+), 22 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index f8bd097733529a..38ed64bd369923 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3981,6 +3981,14 @@ def MatrixType : TypeAttr {
let PragmaAttributeSupport = 0;
}
+def CoopMatrixType : TypeAttr {
+ let Spellings = [Clang<"coop_mat">];
+ let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
+ let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
+ UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
+ let Documentation = [Undocumented];
+}
+
def Visibility : InheritableAttr {
let Clone = 0;
let Spellings = [GCC<"visibility">];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index a4a7fb91e6b39d..489faa0f818f92 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12010,6 +12010,60 @@ def ext_opencl_ext_vector_type_rgba_selector: ExtWarn<
def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
+// OpenCL C Cooperative Matrix.
+def err_invalid_coopmat_attr
+ : Error<"invalid argument of cooperative matrix attribute">;
+
+def err_coop_mat_not_primitive
+ : Error<"cooperative matrix attribute only supported for integer and "
+ "floating-point types">;
+
+def err_coop_mat_rol_col_size
+ : Error<"cooperative matrix row and column must have power-of-two size">;
+
+def err_coop_mem_layout_enum
+ : Error<"memory layout for cooperative matrix should be "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+
+def err_coop_element_and_pointer_type
+ : Error<"inconsistent between cooperative matrix element type and buffer "
+ "pointer type">;
+
+def err_coop_matrix_arg
+ : Error<"argument must be a valid cooperative matrix type">;
+
+def err_coop_matrix_assignment
+ : Error<"builtin return value should be assigned to cooperative matrix "
+ "type variable">;
+
+def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_A">;
+
+def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_B">;
+
+def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+
+def err_coop_matrix_element_type
+ : Error<"inconsistent cooperative matrix element type">;
+
+def err_coop_matrix_row_or_col_mismatch
+ : Error<"mismatch of cooperative matrix row or column">;
+
+def err_coop_matrix_use_type
+ : Error<"inconsistent cooperative matrix use type">;
+
+def err_unsupported_coopmat_binary_operator
+ : Error<"unsupported cooperative matrix binary operator">;
+
+def err_unsupported_coopmat_scalar_operator
+ : Error<"unsupported cooperative matrix scalar operator">;
+
+def err_ref_coopmat_var
+ : Error<"invalid reference of cooperative matrix variable">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 865defa98a1b33..bc491c2541ad51 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3112,6 +3112,34 @@ class Sema final : public SemaBase {
BuiltinCountedByRefKind K);
bool BuiltinCountedByRef(CallExpr *TheCall);
+ // Coop matrix handling.
+ void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc, CallExpr *call);
+ void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc);
+ bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
+ bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult);
+ void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc);
+ ExprResult BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatScalarOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ ExprResult BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+
// Matrix builtin handling.
ExprResult BuiltinMatrixTranspose(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
@@ -15387,8 +15415,10 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
+
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc);
+ SourceLocation AttrLoc, Expr *Scope = nullptr,
+ Expr *Use = nullptr, bool IsCoopMat = false);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
@@ -15411,6 +15441,8 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
+ bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+
/// Build a function type.
///
/// This routine checks the function type according to C++ rules and
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 51506c62d4e35f..ccf377a9a1a671 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4163,6 +4163,27 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
break;
}
+ case Builtin::BIcoop_mat_load:
+ return BuiltinCoopMatrixLoad(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_store:
+ return BuiltinCoopMatrixStore(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_mulAdd:
+ return BuiltinCoopMatrixMulAdd(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ return BuiltinCoopMatrixBinaryOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_mul:
+ return BuiltinCoopMatrixScalarOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_neg:
+ return BuiltinCoopMatrixScalarUnaryOp(TheCall, TheCallResult);
+
case Builtin::BI__builtin_matrix_transpose:
return BuiltinMatrixTranspose(TheCall, TheCallResult);
@@ -17439,6 +17460,270 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
return false;
}
+// Check coop_mat_load/store buffer pointer.
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
+ unsigned PtrArgIdx) {
+ bool ArgError = false;
+ Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
+ ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
+ if (PtrConv.isInvalid())
+ return true;
+ PtrExpr = PtrConv.get();
+ TheCall->setArg(PtrArgIdx, PtrExpr);
+
+ auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
+ QualType ElementTy;
+ if (!PtrTy) {
+ ArgError = true;
+ } else {
+ ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
+ if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ ArgError = true;
+ }
+ }
+
+ if (ArgError) {
+ Diag(PtrExpr->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+ << PtrArgIdx + 1 << 0 << /* pointer to element ty */ 5 << /* no fp */ 0
+ << PtrExpr->getType();
+ }
+
+ return ArgError;
+}
+
+// Check coop_mat_load/store matrix element has same type with buffer pointer.
+void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
+ auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
+ if (!MTy) {
+ Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ return;
+ }
+
+ assert(isa<PointerType>(BufferType));
+ auto *PTy = BufferType->castAs<PointerType>();
+
+ if (MTy->getElementType().getUnqualifiedType() !=
+ PTy->getPointeeType().getUnqualifiedType())
+ Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+}
+
+void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
+
+ FunctionDecl *F = call->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (Fname->isStr("coop_mat_load"))
+ CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
+ MatrixLoc);
+}
+
+// Check coop_mat_load/store layout argument
+bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
+ bool ArgError = false;
+ DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LayoutExpr);
+ if (DR) {
+ const auto *ECDHS = dyn_cast<EnumConstantDecl>(DR->getDecl());
+ if (ECDHS) {
+ if (ECDHS->getInitVal() != 0 && ECDHS->getInitVal() != 1)
+ ArgError = true;
+ } else
+ ArgError = true;
+ } else
+ ArgError = true;
+
+ if (ArgError)
+ Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+
+ return ArgError;
+}
+
+ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
+ return ExprError();
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
+ return ExprError();
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 4))
+ return ExprError();
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ return ExprError();
+ CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
+ return ExprError();
+ return CallResult;
+}
+
+void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+ FunctionDecl *F = TheCall->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (!Fname->isStr("coop_mat_mulAdd"))
+ return;
+
+ auto MC = TheCall->getArg(2);
+ auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = MC->getType()->getAs<CooperativeMatrixType>();
+ auto Loc = TheCall->getBeginLoc();
+
+ if (!MOutTy)
+ Diag(Loc, diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!MOutTy || !M2Ty)
+ return;
+
+ if (MOutTy->getUse() != 2)
+ Diag(Loc, diag::err_coop_matrix_useACC);
+
+ if (MOutTy->getElementType().getUnqualifiedType() !=
+ M2Ty->getElementType().getUnqualifiedType())
+ Diag(Loc, diag::err_coop_matrix_element_type);
+
+ if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+}
+
+bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc) {
+ auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
+ auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
+ if (!M0Ty)
+ Diag(ALoc, diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(BLoc, diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty)
+ return true;
+
+ if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+
+ if (M0Ty->getUse() != M1Ty->getUse()) {
+ Diag(ALoc, diag::err_coop_matrix_use_type);
+ return true;
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType()) {
+ Diag(ALoc, diag::err_coop_matrix_element_type);
+ return true;
+ }
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+
+ CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
+ Arg1->getBeginLoc());
+
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
+ if (ATy->isIntegerType() && CTy->isIntegerType())
+ return true;
+ if (ATy->isFloatingType() && CTy->isFloatingType())
+ return true;
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ Expr *Arg2 = TheCall->getArg(2);
+
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto *M1Ty = Arg1->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = Arg2->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+ auto Loc1 = Arg1->getBeginLoc();
+
+ if (!M0Ty)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty || !M2Ty)
+ return ExprError();
+
+ if (M0Ty->getUse() != 0)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ if (M1Ty->getUse() != 1)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ if (M2Ty->getUse() != 2)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+
+ if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
+ M2Ty->getElementType()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+
+ if (M0Ty->getNumRows() != M2Ty->getNumRows())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
+ (M0Ty->getNumColumns() != M1Ty->getNumRows()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
@@ -17449,8 +17734,9 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
- if (!MType) {
+ auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
+ auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
+ if (!ConstMType && !CoopMType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17459,11 +17745,23 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- QualType ResultType = Context.getConstantMatrixType(
- MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
+ if (ConstMType) {
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
+ if (CoopMType) {
+ QualType ResultType = Context.getCooperativeMatrixType(
+ CoopMType->getElementType(),CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
+
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 4f6e65c1c4708a..232cb88d3eec73 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14135,6 +14135,25 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
+// Return true if RHSExpr is a cooperative matrix builtin call.
+bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ if (!call)
+ return false;
+ FunctionDecl *F = call->getDirectCallee();
+ if (!F)
+ return false;
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ if (!Fname)
+ return false;
+ if (Fname->getName().starts_with("coop_mat") &&
+ !Fname->getName().starts_with("coop_mat_length"))
+ return true;
+
+ return false;
+}
+
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
llvm::scope_exit ResetDeclForInitializer([this]() {
if (!this->ExprEvalContexts.empty())
@@ -14653,6 +14672,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
+
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (!VDecl->getType()->isMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
}
void Sema::ActOnInitializerError(Decl *D) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 2e9a3b73e5c658..340e9789af6d9b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4682,6 +4682,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::TemplateSpecialization:
@@ -8091,6 +8092,16 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
+ if (srcTy->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns();
+ }
+
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -11439,12 +11450,14 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
if (!IsDiv &&
- (LHSTy->isConstantMatrixType() || RHSTy->isConstantMatrixType()))
+ (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+ if (IsDiv && LHSTy->isCooperativeMatrixType() && RHSTy->isArithmeticType())
+ return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
@@ -11844,6 +11857,15 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -11990,6 +12012,15 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -14049,6 +14080,32 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ }
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -15779,6 +15836,47 @@ static bool needsConversionOfHalfVec(bool OpRequiresConversion, ASTContext &Ctx,
return HasVectorOfHalfType(E0) && (!E1 || HasVectorOfHalfType(E1));
}
+ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Add:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
+ Args);
+ case BO_Sub:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
+ Args);
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
+ Args);
+ case BO_Div:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_binary_operator));
+}
+
+ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_scalar_operator));
+}
+
ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
BinaryOperatorKind Opc, Expr *LHSExpr,
Expr *RHSExpr, bool ForFoldExpression) {
@@ -15815,6 +15913,17 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
if (getLangOpts().OpenCL) {
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
+ // Cooperative matrix support
+ if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ // Check matrix types for assignment.
+ if (BO_Assign == Opc) {
+ if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
+ RHSExpr->getBeginLoc()))
+ return ExprError();
+ } else
+ return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
@@ -15842,6 +15951,18 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (!LHSExpr->getType()->isMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
+ return ExprError();
+ }
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ assert(call);
+ call->setType(LHSExpr->getType());
+ CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ call);
+ CheckCoopMatrixMatMulOutput(call);
+ }
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
LHS.get()->getObjectKind() != OK_ObjCProperty) {
@@ -16501,6 +16622,10 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
+ if (Opc == UO_Minus && Ty->isMatrixType()) {
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
+ {InputExpr});
+ }
// The only legal unary operation for atomics is '&'.
if ((Opc != UO_AddrOf && Ty->isAtomicType()) ||
// OpenCL special types - image, sampler, pipe, and blocks are to be used
@@ -16579,7 +16704,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
Opc == UO_Plus && resultType->isPointerType())
break;
-
+ else if (getLangOpts().OpenCL && Opc == UO_Minus &&
+ resultType->isCooperativeMatrixType())
+ break;
return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
<< resultType << Input.get()->getSourceRange());
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 75cad0bb433f89..bc3f12b3167320 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3267,6 +3267,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BitInt:
break;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 5a61cb852d15e5..eebca2d253093b 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6416,6 +6416,11 @@ bool UnnamedLocalNoLinkageFinder::VisitConstantMatrixType(
return Visit(T->getElementType());
}
+bool UnnamedLocalNoLinkageFinder::VisitCooperativeMatrixType(
+ const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+}
+
bool UnnamedLocalNoLinkageFinder::VisitFunctionProtoType(
const FunctionProtoType* T) {
for (const auto &A : T->param_types()) {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 90724e1d01cb20..284561435727e8 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2455,6 +2455,30 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
&DependentSizedMatrixType::getColumnExpr);
}
+ // (clang extension)
+ //
+ // T __attribute__((coop_mat_type(<integral constant>,
+ // <integral constant>,
+ // <integral constant>,
+ // <integral constant>)))
+ case Type::CooperativeMatrix: {
+ const auto *MP = P->castAs<CooperativeMatrixType>(),
+ *MA = A->getAs<CooperativeMatrixType>();
+ if (!MA)
+ return TemplateDeductionResult::NonDeducedMismatch;
+
+ // Check that the dimensions are the same
+ if (MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns()) {
+ return TemplateDeductionResult::NonDeducedMismatch;
+ }
+ // Perform deduction on element types.
+ return DeduceTemplateArgumentsByTypeMatch(
+ S, TemplateParams, MP->getElementType(), MA->getElementType(), Info,
+ Deduced, TDF, degradeCallPartialOrderingKind(POK),
+ /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
+ }
+
// (clang extension)
//
// T __attribute__(((address_space(N))))
@@ -7103,6 +7127,13 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *MatType = cast<CooperativeMatrixType>(T);
+ MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
+ Depth, Used);
+ break;
+ }
+
case Type::DependentSizedMatrix: {
const DependentSizedMatrixType *MatType = cast<DependentSizedMatrixType>(T);
MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 3d444bfabc8c39..36619f92c82964 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2532,9 +2532,11 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc) {
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc, Expr *ScopeExpr,
+ Expr *UseExpr, bool IsCoopMat) {
+ if (!IsCoopMat)
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2615,6 +2617,26 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
+ if (IsCoopMat) {
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr)
+ << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
@@ -5978,6 +6000,23 @@ static void fillAtomicQualLoc(AtomicTypeLoc ATL, const DeclaratorChunk &Chunk) {
ATL.setParensRange(SourceRange());
}
+static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
+ const ParsedAttributesView &Attrs) {
+ for (const ParsedAttr &AL : Attrs) {
+ if (AL.getKind() == ParsedAttr::AT_CoopMatrixType) {
+ MTL.setAttrNameLoc(AL.getLoc());
+ MTL.setAttrScopeOperand(AL.getArgAsExpr(0));
+ MTL.setAttrRowOperand(AL.getArgAsExpr(1));
+ MTL.setAttrColumnOperand(AL.getArgAsExpr(2));
+ MTL.setAttrUseOperand(AL.getArgAsExpr(3));
+ MTL.setAttrOperandParensRange(SourceRange());
+ return;
+ }
+ }
+
+ llvm_unreachable("no matrix_type attribute found at the expected location!");
+}
+
namespace {
class TypeSpecLocFiller : public TypeLocVisitor<TypeSpecLocFiller> {
Sema &SemaRef;
@@ -6373,6 +6412,9 @@ namespace {
void VisitMatrixTypeLoc(MatrixTypeLoc TL) {
fillMatrixTypeLoc(TL, Chunk.getAttrs());
}
+ void VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
+ fillCooperativeMatrixTypeLoc(TL, Chunk.getAttrs());
+ }
void VisitTypeLoc(TypeLoc TL) {
llvm_unreachable("unsupported TypeLoc kind in declarator!");
@@ -8988,23 +9030,28 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S) {
- if (!S.getLangOpts().MatrixTypes) {
+ Sema &S, bool IsCoopMat = false) {
+ if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- if (Attr.getNumArgs() != 2) {
+ unsigned int NumAttrs = IsCoopMat ? 4 : 2;
+ if (Attr.getNumArgs() != NumAttrs) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << 2;
+ << Attr << NumAttrs;
return;
}
-
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ if (IsCoopMat) {
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
+ Scope, Use, /* IsCoopMat */ true);
+ if (!T.isNull())
+ CurType = T;
+ }
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9288,6 +9335,11 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
attr.setUsedAsTypeAttr();
break;
+ case ParsedAttr::AT_CoopMatrixType:
+ HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ attr.setUsedAsTypeAttr();
+ break;
+
case ParsedAttr::AT_WebAssemblyFuncref: {
if (!HandleWebAssemblyFuncrefAttr(state, type, attr))
attr.setUsedAsTypeAttr();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 25e9113aeb7a46..f4547eb6453ebf 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1048,6 +1048,12 @@ class TreeTransform {
unsigned NumColumns,
SourceLocation AttributeLoc);
+ /// Build a new cooperative matrix type given the element type and
+ /// scope and use and dimensions.
+ QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use);
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
@@ -6310,6 +6316,34 @@ TreeTransform<Derived>::TransformConstantMatrixType(TypeLocBuilder &TLB,
return Result;
}
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformCooperativeMatrixType(
+ TypeLocBuilder &TLB, CooperativeMatrixTypeLoc TL) {
+ const CooperativeMatrixType *T = TL.getTypePtr();
+ QualType ElementType = getDerived().TransformType(T->getElementType());
+ if (ElementType.isNull())
+ return QualType();
+
+ QualType Result = TL.getType();
+ if (getDerived().AlwaysRebuild() || ElementType != T->getElementType()) {
+ Result = getDerived().RebuildCooperativeMatrixType(
+ ElementType, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ if (Result.isNull())
+ return QualType();
+ }
+
+ CooperativeMatrixTypeLoc NewTL = TLB.push<CooperativeMatrixTypeLoc>(Result);
+ NewTL.setAttrNameLoc(TL.getAttrNameLoc());
+ NewTL.setAttrOperandParensRange(TL.getAttrOperandParensRange());
+ NewTL.setAttrScopeOperand(TL.getAttrScopeOperand());
+ NewTL.setAttrRowOperand(TL.getAttrRowOperand());
+ NewTL.setAttrColumnOperand(TL.getAttrColumnOperand());
+ NewTL.setAttrUseOperand(TL.getAttrUseOperand());
+
+ return Result;
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::TransformDependentSizedMatrixType(
TypeLocBuilder &TLB, DependentSizedMatrixTypeLoc TL) {
@@ -18237,6 +18271,14 @@ QualType TreeTransform<Derived>::RebuildConstantMatrixType(
AttributeLoc);
}
+template <typename Derived>
+QualType TreeTransform<Derived>::RebuildCooperativeMatrixType(
+ QualType ElementType, unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use) {
+ return SemaRef.Context.getCooperativeMatrixType(ElementType, Scope, NumRows,
+ NumColumns, Use);
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::RebuildDependentSizedMatrixType(
QualType ElementType, Expr *RowExpr, Expr *ColumnExpr,
>From 381ea840d7ce927631cca55ce1537b0b92441f84 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:56 -0400
Subject: [PATCH 05/48] Patch 5 - Emit LLVM IR and intrinsics
---
clang/include/clang/Basic/Builtins.td | 67 +++++++
clang/lib/CodeGen/CGBuiltin.cpp | 277 ++++++++++++++++++++++++++
clang/lib/CodeGen/CGDebugInfo.cpp | 29 +++
clang/lib/CodeGen/CGDebugInfo.h | 1 +
clang/lib/CodeGen/CGExpr.cpp | 61 ++++++
clang/lib/CodeGen/CGExprScalar.cpp | 14 ++
clang/lib/CodeGen/CodeGenFunction.cpp | 2 +
clang/lib/CodeGen/CodeGenFunction.h | 9 +
clang/lib/CodeGen/CodeGenTBAA.cpp | 2 +-
clang/lib/CodeGen/CodeGenTypes.cpp | 33 ++-
clang/lib/CodeGen/ItaniumCXXABI.cpp | 2 +
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +
clang/lib/CodeGen/QualTypeMapper.h | 1 +
13 files changed, 502 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index ad9a58bb2ad2d3..067c112c23ff71 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,6 +1931,73 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
+// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+def CoopMatLoad : Builtin {
+ let Spellings = ["coop_mat_load"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatStore : Builtin {
+ let Spellings = ["coop_mat_store"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatMulAdd : Builtin {
+ let Spellings = ["coop_mat_mulAdd"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryAdd : Builtin {
+ let Spellings = ["coop_mat_binary_add"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinarySub : Builtin {
+ let Spellings = ["coop_mat_binary_sub"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryMul : Builtin {
+ let Spellings = ["coop_mat_binary_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryDiv : Builtin {
+ let Spellings = ["coop_mat_binary_div"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarMul : Builtin {
+ let Spellings = ["coop_mat_scalar_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarNeg : Builtin {
+ let Spellings = ["coop_mat_scalar_neg"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatInit : Builtin {
+ let Spellings = ["coop_mat_init"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatLength : Builtin {
+ let Spellings = ["coop_mat_length"];
+ let Attributes = [NoThrow, Const, CustomTypeChecking];
+ let Prototype = "unsigned int(...)";
+}
+
// "Overloaded" Atomic operator builtins. These are overloaded to support data
// types of i8, i16, i32, i64, and i128. The front-end sees calls to the
// non-suffixed version of these (which has a bogus type) and transforms them to
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index c93375ff3a18ef..9e215d7b19adad 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2714,6 +2714,54 @@ static void ClearPadding(CodeGenFunction &CGF, Address Src,
}
}
+static std::string getSPIRVBuiltinName(unsigned BuiltinID,
+ bool IsFloat = false) {
+ switch (BuiltinID) {
+ case Builtin::BIcoop_mat_load:
+ return "__spirv_CooperativeMatrixLoadKHR";
+ case Builtin::BIcoop_mat_store:
+ return "__spirv_CooperativeMatrixStoreKHR";
+ case Builtin::BIcoop_mat_mulAdd:
+ return "__spirv_CooperativeMatrixMulAddKHR";
+ case Builtin::BIcoop_mat_binary_add:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFAdd" : "__spirv_IAdd";
+ case Builtin::BIcoop_mat_binary_sub:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFSub" : "__spirv_ISub";
+ case Builtin::BIcoop_mat_binary_mul:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFMul" : "__spirv_IMul";
+ case Builtin::BIcoop_mat_binary_div:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFDiv" : "__spirv_IDiv";
+ case Builtin::BIcoop_mat_scalar_mul:
+ return "__spirv_CooperativeMatrixScalarMulKHR";
+ case Builtin::BIcoop_mat_scalar_neg:
+ return "__spirv_CooperativeMatrixScalarNeg";
+ case Builtin::BIcoop_mat_init:
+ return "__spirv_CompositeConstruct";
+ case Builtin::BIcoop_mat_length:
+ return "__spirv_CooperativeMatrixLengthKHR";
+ }
+ assert(0 && "Unexpected Builtin");
+ return "";
+}
+
+static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
+ CodeGenModule &CGM,
+ const CooperativeMatrixType *MTy) {
+ llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
+ MTy->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
+} // namespace
+
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
@@ -4575,6 +4623,235 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(Result);
}
+ case Builtin::BIcoop_mat_load: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address SrcAddr = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(SrcAddr, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_load without coop_mat output operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Layout = EmitScalarExpr(E->getArg(1));
+ auto Stride = EmitScalarExpr(E->getArg(2));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_store: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address Dst = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(Dst, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_store without coop_mat input operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Arg0 = EmitScalarExpr(E->getArg(1));
+ auto Layout = EmitScalarExpr(E->getArg(2));
+ auto Stride = EmitScalarExpr(E->getArg(3));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_mulAdd: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_mulAdd without coop_mat output operand");
+
+ auto Arg0 = E->getArg(0);
+ auto Arg1 = E->getArg(1);
+ auto Arg2 = E->getArg(2);
+ auto MA = EmitScalarExpr(Arg0);
+ auto MB = EmitScalarExpr(Arg1);
+ auto MC = EmitScalarExpr(Arg2);
+ const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
+ const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MATy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *CType = getTargetExtType(*this, CGM, MCTy);
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, CType, Int1Ty}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ llvm::Type *ElTy = ConvertType(MTy->getElementType());
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name = (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_mul: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = Arg1->getType();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_neg: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_init: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ auto Init = EmitScalarExpr(E->getArg(0));
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Init->getType()}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(InitFn, {Init});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_length: {
+ const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ ConvertType(E->getType()), {AType}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
case Builtin::BI__builtin_masked_load:
case Builtin::BI__builtin_masked_expand_load: {
llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 93428777b9b24c..408fd1ec25a1ef 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3937,6 +3937,33 @@ llvm::DIType *CGDebugInfo::CreateType(const ConstantMatrixType *Ty,
return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
}
+llvm::DIType *CGDebugInfo::CreateType(const CooperativeMatrixType *Ty,
+ llvm::DIFile *Unit) {
+ // FIXME: Create another debug type for matrices
+ // For the time being, it treats it like a nested ArrayType.
+
+ llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
+ uint64_t Size = CGM.getContext().getTypeSize(Ty);
+ uint32_t Align = getTypeAlignIfRequired(Ty, CGM.getContext());
+
+ // Create ranges for both dimensions.
+ llvm::SmallVector<llvm::Metadata *, 2> Subscripts;
+ auto *ColumnCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumColumns()));
+ auto *RowCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumRows()));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ ColumnCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ RowCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
+ return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
+}
+
llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
uint64_t Size;
uint32_t Align;
@@ -4396,6 +4423,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
return CreateType(cast<VectorType>(Ty), Unit);
case Type::ConstantMatrix:
return CreateType(cast<ConstantMatrixType>(Ty), Unit);
+ case Type::CooperativeMatrix:
+ return CreateType(cast<CooperativeMatrixType>(Ty), Unit);
case Type::ObjCObjectPointer:
return CreateType(cast<ObjCObjectPointerType>(Ty), Unit);
case Type::ObjCObject:
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 37520394812b44..f617623af3af40 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -244,6 +244,7 @@ class CGDebugInfo {
llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ConstantMatrixType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const CooperativeMatrixType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 96c6022f599b77..f794cd2c9eb8fc 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7654,3 +7654,64 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
}
+
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
+ llvm::Value *ScalarVal,
+ QualType CoopMatQTy)
+{
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(
+ VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(
+ VecTy->getElementCount(),
+ ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
+ BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy)
+{
+ auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
+
+ switch (Opcode) {
+ case BO_Add:
+ return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
+ : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+
+ case BO_Sub:
+ return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
+ : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+
+ case BO_Mul:
+ // mat * mat → element-wise FMul/IMul
+ // mat * scalar → handled separately (see scalar path below)
+ if (!RHS->getType()->isVectorTy()) {
+ // mat * scalar: broadcast scalar then element-wise mul
+ llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
+ return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
+ : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ }
+ return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
+ : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+
+ case BO_Div:
+ if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+
+ default:
+ llvm_unreachable("Unsupported cooperative matrix binary op");
+ }
+}
\ No newline at end of file
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index f1394fc5246459..0323398091a690 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -956,6 +956,12 @@ class ScalarExprEmitter
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
+ if (E->getType()->isCooperativeMatrixType()) { \
+ return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
+ CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), \
+ E->getType()); \
+ } \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
if (result && !promotionTy.isNull()) \
@@ -3687,6 +3693,14 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
+ QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ if (CompTy->isFloatingType())
+ return Builder.CreateFNeg(Val, "coopmat.fneg");
+ else
+ return Builder.CreateNeg(Val, "coopmat.ineg");
+ }
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index e5bca359e6a855..4dcb86212cb093 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -261,6 +261,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -2588,6 +2589,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 73090c03484b73..6604a46f43dccd 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5192,6 +5192,15 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
+ /// Helper function for EmitCoopMatBinaryOp
+ llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index d33188db9da158..c0ba3582076415 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -332,7 +332,7 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert(isa<ConstantMatrixType>(Ty) &&
+ assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index daed8559cc62f5..6ac24c6554922e 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -125,7 +125,21 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
-
+ if (T->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent a Coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+ return CoopMatType;
+ }
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -704,6 +718,23 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
MT->getNumRows() * MT->getNumColumns());
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+
+ ResultType = CoopMatType;
+ break;
+ }
case Type::FunctionNoProto:
case Type::FunctionProto:
ResultType = ConvertFunctionTypeInternal(T);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 5bbb742a7b1efa..e0f2dce9077c9d 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3713,6 +3713,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -4016,6 +4017,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 5044883a49ca2e..5850604f88207b 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -136,6 +136,12 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
MT->getNumRows() * MT->getNumColumns(),
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(QT);
+ return Builder.getArrayType(convertType(MT->getElementType()),
+ MT->getNumRows() * MT->getNumColumns(),
+ ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ }
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
case Type::BitInt: {
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index f6bb43482c705e..73d1753349fd40 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -48,6 +48,7 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
+ const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
>From fa1889581230425a516867cb7382c5952e1b0a25 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:20:46 -0400
Subject: [PATCH 06/48] Add LIT tests and fix minor build issues
---
clang/include/clang/AST/TypeBase.h | 8 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 94 +++++++++++++++
.../test/Preprocessor/coop-mat-opencl-ext.cl | 63 ++++++++++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 66 ++++++++++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 60 +++++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 83 +++++++++++++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 114 ++++++++++++++++++
clang/tools/libclang/CIndex.cpp | 1 +
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ++++++
9 files changed, 524 insertions(+), 4 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen.cl
create mode 100644 clang/test/Preprocessor/coop-mat-opencl-ext.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-ast-utils.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema-neg.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-type-infra.cl
create mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 9b1c9ef646f1f5..b8146904612239 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4627,14 +4627,14 @@ class CooperativeMatrixType final : public MatrixType {
/// Return true if \p Scope is valid
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
}
/// Return true if \p Use is valid
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
}
/// Returns the maximum number of elements per dimension.
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
new file mode 100644
index 00000000000000..58b7a882fdc870
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -0,0 +1,94 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+//
+// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
+// SPIR-V intrinsic names.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
+// TargetExtType (visible in the call signature).
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// ---------------------------------------------------------------------------
+kernel void test_store(__global float *ptr, MatA_t a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
+
+// ---------------------------------------------------------------------------
+// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a; MatB_t b; MatC_t c; MatC_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+
+// ---------------------------------------------------------------------------
+// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// ---------------------------------------------------------------------------
+kernel void test_binary_add(__global float *ptr) {
+ MatA_t a; MatA_t b; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a + b;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_binary_add
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
+
+// ---------------------------------------------------------------------------
+// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// ---------------------------------------------------------------------------
+kernel void test_scalar_mul(__global float *ptr, float s) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a * s;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+
+// ---------------------------------------------------------------------------
+// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// ---------------------------------------------------------------------------
+kernel void test_unary_neg(__global float *ptr) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = -a;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_unary_neg
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarNeg
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
new file mode 100644
index 00000000000000..7181efbe7e2a88
--- /dev/null
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -0,0 +1,63 @@
+// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+//
+// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
+//
+// Tests: extension macro is predefined when enabled, extension can be
+// explicitly enabled/disabled via pragma, all four enum types and
+// their constants are visible under -finclude-default-header.
+
+// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E -dM %s \
+// RUN: | FileCheck %s --check-prefix=EXT
+
+// EXT: cl_ext_kernel_cooperative_matrix
+
+// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E %s \
+// RUN: | FileCheck %s --check-prefix=NOEXT
+
+// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+
+// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+void test_enum_constants(void) {
+ // coop_matrix_scope_t
+ coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
+ (void)s;
+
+ // coop_matrix_use_t
+ coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
+ coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
+ coop_matrix_use_t u2 = CLK_COOPERATIVE_MATRIX_ACCUMULATOR;
+ (void)u0; (void)u1; (void)u2;
+
+ // coop_matrix_layout_t
+ coop_matrix_layout_t l0 = CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR;
+ coop_matrix_layout_t l1 = CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR;
+ (void)l0; (void)l1;
+
+ // coop_matrix_operands_t
+ coop_matrix_operands_t op = CLK_COOPERATIVE_MATRIX_OPERAND_NONE;
+ (void)op;
+}
+
+// ── 2d. Enum constant values match the spec ──────────────────────────────────
+void test_enum_values(void) {
+ _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR == 0, "row major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR== 1, "col major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_OPERAND_NONE == 0, "operand none");
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
new file mode 100644
index 00000000000000..c6230eb94531f4
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -0,0 +1,66 @@
+// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
+//
+// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
+// ExprConstant, ItaniumMangle, MicrosoftMangle,
+// ASTReader/ASTWriter TypeLoc serialisation.
+//
+// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefix=MANGLE
+//
+// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch \
+// RUN: -ast-dump %t.pch_probe.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+//
+// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
+kernel void test_mangling(MatA_t a) { (void)a; }
+
+// MANGLE: @{{.*}}test_mangling{{.*}}(
+
+// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
+// must still carry a coop_mat parameter type.
+// PCH: FunctionDecl {{.*}} pch_probe
+// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
+
+// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_structural_equiv(void) {
+ MatA_t *p = 0;
+ MatA_alias *q = p; // same canonical type
+ (void)q;
+}
+
+// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
+// integer, float, pointer …). Using __builtin_classify_type on it
+// compiles without error; result is 0 (no_type_class).
+void test_expr_constant(MatA_t a) {
+ int cls = __builtin_classify_type(a);
+ (void)cls;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
new file mode 100644
index 00000000000000..f8080448b6df51
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -0,0 +1,60 @@
+// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+//
+// Patch 4: Sema — negative tests for diagnostic paths.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 2. Invalid use value (99 is not 0/1/2)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 3. Mismatched element types in coop_mat_mulAdd
+// a/b are float matrices, c is an int matrix — should fire element type
+// mismatch diagnostic.
+// Use the two-step declare-then-assign pattern so that the coop_mat_load
+// fixup path fires correctly and we only get the intended mulAdd error.
+// ---------------------------------------------------------------------------
+kernel void test_muladd_type_mismatch(__global float *fptr,
+ __global int *iptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+ a = coop_mat_load(fptr, ROW_MAJOR, 16);
+ b = coop_mat_load(fptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+
+ MatC_int_t result;
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
+// This intentionally uses single-step initialization because the error
+// fires precisely when the LHS is NOT a coop mat type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_assignment(__global float *ptr) {
+ float bad;
+ bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
+ (void)bad;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
new file mode 100644
index 00000000000000..cec1f278b0037a
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -0,0 +1,83 @@
+// clang/test/SemaOpenCL/coop-mat-sema.cl
+//
+// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
+// builtin validation, expr handling, TreeTransform.
+//
+// Valid code -- expected-no-diagnostics
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
+// match the LHS variable type. Declare first, then assign so that
+// Sema's IsCoopMatrixBuiltin path fires correctly.
+// ---------------------------------------------------------------------------
+kernel void test_load_store(__global float *ptr,
+ __global float *out_ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ coop_mat_store(out_ptr, a, ROW_MAJOR, 16);
+}
+
+// ---------------------------------------------------------------------------
+// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ MatC_t result;
+ result = coop_mat_mulAdd(a, b, c);
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4d. Binary element-wise operators (+, -)
+// ---------------------------------------------------------------------------
+void test_binary_ops(MatA_t a, MatA_t b) {
+ MatA_t r_add = a + b;
+ MatA_t r_sub = a - b;
+ (void)r_add; (void)r_sub;
+}
+
+// ---------------------------------------------------------------------------
+// 4e. Scalar multiply operator
+// ---------------------------------------------------------------------------
+void test_scalar_ops(MatA_t a, float s) {
+ MatA_t r = a * s;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4f. Unary minus
+// ---------------------------------------------------------------------------
+void test_unary_minus(MatA_t a) {
+ MatA_t r = -a;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
+// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// ---------------------------------------------------------------------------
+kernel void test_assignment(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
new file mode 100644
index 00000000000000..2e6700973e6113
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -0,0 +1,114 @@
+// clang/test/SemaOpenCL/coop_mat_type_infra.cl
+//
+// Patch 1: CooperativeMatrixType AST node and type infrastructure.
+//
+// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
+// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
+// RecursiveASTVisitor traversal, mergeTypes compatibility,
+// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -ast-dump %s \
+// RUN: | FileCheck %s --check-prefix=AST
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+
+// ---------------------------------------------------------------------------
+// Enum constants (from opencl-c-base.h via -finclude-default-header)
+// ---------------------------------------------------------------------------
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+// ---------------------------------------------------------------------------
+// 1. Basic type construction — four use roles, float element type.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_float16x16;
+typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
+
+// AST: TypedefDecl {{.*}} MatA_float16x16
+// AST: TypedefDecl {{.*}} MatB_float16x16
+// AST: TypedefDecl {{.*}} MatC_float16x16
+// AST: TypedefDecl {{.*}} MatA_int8x8
+
+// ---------------------------------------------------------------------------
+// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// ---------------------------------------------------------------------------
+void test_type_spelling(void) {
+ MatA_float16x16 a;
+ MatB_float16x16 b;
+ MatC_float16x16 c;
+ MatA_int8x8 d;
+}
+
+// AST: VarDecl {{.*}} a {{.*}}coop_mat(
+// AST: VarDecl {{.*}} b {{.*}}coop_mat(
+// AST: VarDecl {{.*}} c {{.*}}coop_mat(
+// AST: VarDecl {{.*}} d {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
+// float(4B)*16*16 = 1024 B = 8192 bits
+// int(4B)*8*8 = 256 B = 2048 bits
+// ---------------------------------------------------------------------------
+void test_sizeof(void) {
+ _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
+ _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
+}
+
+// ---------------------------------------------------------------------------
+// 4. Parameter / return type — type preserved through function boundary.
+// ---------------------------------------------------------------------------
+MatA_float16x16 test_param_return(MatA_float16x16 in) {
+ return in;
+}
+
+// AST: FunctionDecl {{.*}} test_param_return
+// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 5. TypeLoc operand traversal — all four operand slots populated.
+// ---------------------------------------------------------------------------
+void test_typeloc_operands(void) {
+ float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
+ (void)local_acc;
+}
+
+// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// ---------------------------------------------------------------------------
+void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
+ (void)x;
+}
+
+// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_merge_types(void) {
+ MatA_float16x16 *p = 0;
+ MatA_alias *q = p; // same canonical type — no diagnostic
+ (void)q;
+}
+
+// ---------------------------------------------------------------------------
+// 8. PCH serialisation round-trip.
+// ---------------------------------------------------------------------------
+// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 8d246d41f31f66..951fedea44ee4d 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1912,6 +1912,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
new file mode 100644
index 00000000000000..c4e206b76b7603
--- /dev/null
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -0,0 +1,39 @@
+; llvm/test/Verifier/coop_mat_verifier_fix.ll
+;
+; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
+;
+; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
+; ((A && B) <= 64). After the fix the parentheses are explicit.
+;
+; This test verifies that a valid @llvm.experimental.noalias.scope.decl
+; call with a 2-element scope list (no third argument) passes the verifier
+; without error, and that an invalid third argument (> 64 bits) is caught.
+;
+; ── 6a. Valid call — no third argument → verifier must accept ────────────────
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
+; RUN: llvm-as < %s | opt -passes=verify -disable-output
+;
+; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
+; RUN: llvm-as %s -o %t.bc
+; RUN: opt -passes=verify %t.bc -disable-output
+
+; VALID: @llvm.experimental.noalias.scope.decl
+
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+define void @test_no_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+; Third arg present and <= 64 bits wide — must also pass
+define void @test_valid_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain_a"}
>From bd9526ffe5b050ab9c3130180806ae780bb43afa Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 18:39:15 -0400
Subject: [PATCH 07/48] Fix formatting issues
---
clang/include/clang/Sema/Sema.h | 15 +++---
clang/lib/CodeGen/CGBuiltin.cpp | 76 +++++++++++++++------------
clang/lib/CodeGen/CGExpr.cpp | 63 ++++++++++------------
clang/lib/CodeGen/CGExprScalar.cpp | 12 ++---
clang/lib/Sema/SemaChecking.cpp | 39 +++++++-------
clang/lib/Sema/SemaExpr.cpp | 35 ++++++------
clang/lib/Sema/TreeTransform.h | 2 +-
clang/lib/Serialization/ASTReader.cpp | 3 +-
clang/lib/Serialization/ASTWriter.cpp | 3 +-
9 files changed, 120 insertions(+), 128 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index bc491c2541ad51..61ccd3ad7f2185 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3114,19 +3114,16 @@ class Sema final : public SemaBase {
// Coop matrix handling.
void CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc, CallExpr *call);
+ SourceLocation MatrixLoc, CallExpr *call);
void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc);
+ QualType BufferType,
+ SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
- ExprResult CallResult);
- ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
- ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 9e215d7b19adad..da615ae22d45dd 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2752,7 +2752,7 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
llvm::Type *Tys[] = {ElTy};
// Unsigned arguments for TargetExtType
unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
+ MTy->getUse()};
// Create a TargetExtType to represent the coop matrix type
llvm::TargetExtType *RetType = llvm::TargetExtType::get(
CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
@@ -4632,8 +4632,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_load without coop_mat output operand");
+ CGM.ErrorUnsupported(E, "coop_mat_load without coop_mat output operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Layout = EmitScalarExpr(E->getArg(1));
@@ -4647,7 +4646,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LoadFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
@@ -4664,8 +4664,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_store without coop_mat input operand");
+ CGM.ErrorUnsupported(E, "coop_mat_store without coop_mat input operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Arg0 = EmitScalarExpr(E->getArg(1));
@@ -4675,11 +4674,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
- VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
+ false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee StoreFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
@@ -4690,8 +4691,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_mulAdd: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_mulAdd without coop_mat output operand");
+ CGM.ErrorUnsupported(E,
+ "coop_mat_mulAdd without coop_mat output operand");
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
@@ -4715,11 +4716,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto *AType = getTargetExtType(*this, CGM, MATy);
auto *BType = getTargetExtType(*this, CGM, MBTy);
auto *CType = getTargetExtType(*this, CGM, MCTy);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee MatMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
@@ -4733,7 +4735,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_binary_div: {
auto Arg0 = EmitScalarExpr(E->getArg(0));
auto Arg1 = EmitScalarExpr(E->getArg(1));
-
+
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
@@ -4750,13 +4752,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
-
- std::string Name = (ElTy->isIntegerTy())
- ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
- : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
- llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name =
+ (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
@@ -4782,10 +4786,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
@@ -4809,10 +4814,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarNegFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
@@ -4825,10 +4831,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Init = EmitScalarExpr(E->getArg(0));
auto *RetType = getTargetExtType(*this, CGM, MTy);
// Set function type.
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {Init->getType()}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {Init->getType()}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee InitFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(InitFn, {Init});
@@ -4840,11 +4847,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
auto *AType = getTargetExtType(*this, CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- ConvertType(E->getType()), {AType}, false);
-
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
+
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LengthFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index f794cd2c9eb8fc..c8363095fc5457 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7655,44 +7655,37 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
- llvm::Value *ScalarVal,
- QualType CoopMatQTy)
-{
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
-
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(
- VecTy->getElementCount(), C);
-
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(
- VecTy->getElementCount(),
- ScalarVal,
- "coopmat.broadcast");
-}
-
-llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
- BinaryOperatorKind Opcode,
- llvm::Value *LHS,
- llvm::Value *RHS,
- QualType ResultTy)
-{
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy) {
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy) {
auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
switch (Opcode) {
case BO_Add:
return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+ : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
case BO_Sub:
return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+ : Builder.CreateSub(LHS, RHS, "coopmat.isub");
case BO_Mul:
// mat * mat → element-wise FMul/IMul
@@ -7701,14 +7694,16 @@ llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
// mat * scalar: broadcast scalar then element-wise mul
llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
}
return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+ : Builder.CreateMul(LHS, RHS, "coopmat.imul");
case BO_Div:
- if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ if (IsFloat)
+ return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned)
+ return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
default:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 0323398091a690..ac37f3da397ffa 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -957,10 +957,9 @@ class ScalarExprEmitter
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
if (E->getType()->isCooperativeMatrixType()) { \
- return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
- CGF.EmitScalarExpr(E->getLHS()), \
- CGF.EmitScalarExpr(E->getRHS()), \
- E->getType()); \
+ return CGF.EmitCoopMatBinaryOp( \
+ E->getOpcode(), CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), E->getType()); \
} \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
@@ -3695,11 +3694,12 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ QualType CompTy =
+ E->getType()->getAs<CooperativeMatrixType>()->getElementType();
if (CompTy->isFloatingType())
return Builder.CreateFNeg(Val, "coopmat.fneg");
else
- return Builder.CreateNeg(Val, "coopmat.ineg");
+ return Builder.CreateNeg(Val, "coopmat.ineg");
}
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index ccf377a9a1a671..9e8c42975f025a 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17461,8 +17461,7 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
}
// Check coop_mat_load/store buffer pointer.
-bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
- unsigned PtrArgIdx) {
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
bool ArgError = false;
Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
@@ -17493,8 +17492,8 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
// Check coop_mat_load/store matrix element has same type with buffer pointer.
void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc) {
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
@@ -17510,8 +17509,8 @@ void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
}
void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc,
- CallExpr *call) {
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
FunctionDecl *F = call->getDirectCallee();
assert(F);
@@ -17520,7 +17519,7 @@ void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
assert(Fname);
if (Fname->isStr("coop_mat_load"))
CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ MatrixLoc);
}
// Check coop_mat_load/store layout argument
@@ -17544,7 +17543,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
}
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
@@ -17563,7 +17562,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
return CallResult;
@@ -17602,7 +17601,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
- SourceLocation BLoc) {
+ SourceLocation BLoc) {
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
@@ -17631,7 +17630,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
}
ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17639,7 +17638,7 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ Arg1->getBeginLoc());
TheCall->setType(Arg0->getType());
@@ -17655,7 +17654,7 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
}
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
@@ -17703,7 +17702,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17714,7 +17713,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
return ExprError();
@@ -17747,17 +17746,17 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// matrix type.
if (ConstMType) {
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
}
if (CoopMType) {
QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(),CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
+ CoopMType->getElementType(), CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 340e9789af6d9b..c056a224a91256 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -11449,8 +11449,7 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
if (LHSTy->isSveVLSBuiltinType() || RHSTy->isSveVLSBuiltinType())
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
- if (!IsDiv &&
- (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
+ if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
@@ -14081,10 +14080,11 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return QualType();
if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
+ RHS.get()->getType()->isCooperativeMatrixType()) {
auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a matrix");
if (LHSMatType && RHSMatType) {
if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
return InvalidOperands(Loc, LHS, RHS);
@@ -14095,14 +14095,14 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
RHS.get()->getType().getUnqualifiedType());
QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
+ RHSELTy = RHSMatType->getElementType();
if (!Context.hasSameType(LHSELTy, RHSELTy))
return InvalidOperands(Loc, LHS, RHS);
return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
- LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
- LHSMatType->getUse());
+ Context.getCommonSugaredType(LHSELTy, RHSELTy),
+ LHSMatType->getScope(), LHSMatType->getNumRows(),
+ RHSMatType->getNumColumns(), LHSMatType->getUse());
}
}
@@ -15844,17 +15844,13 @@ ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Add:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add, Args);
case BO_Sub:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub, Args);
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul, Args);
case BO_Div:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div, Args);
default:
break;
}
@@ -15869,8 +15865,7 @@ ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul, Args);
default:
break;
}
@@ -15918,7 +15913,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
- RHSExpr->getBeginLoc()))
+ RHSExpr->getBeginLoc()))
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
@@ -15960,7 +15955,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
assert(call);
call->setType(LHSExpr->getType());
CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
+ call);
CheckCoopMatrixMatMulOutput(call);
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index f4547eb6453ebf..55f6c930625388 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1053,7 +1053,7 @@ class TreeTransform {
QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
unsigned NumRows, unsigned NumColumns,
unsigned Use);
-
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 9d8fea3651cdf4..33c44b628a5a28 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7693,8 +7693,7 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
-void TypeLocReader::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
TL.setAttrOperandParensRange(readSourceRange());
TL.setAttrScopeOperand(Reader.readExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 42d6f33e0711e1..33713ff3967668 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -457,8 +457,7 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
-void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
SourceRange range = TL.getAttrOperandParensRange();
addSourceLocation(range.getBegin());
>From b38533e6093057fc74d5dec111c87a26c4a8d15c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 15 Jul 2026 12:12:32 -0400
Subject: [PATCH 08/48] Add tests for coop_mat_init and coop_mat_length
---
.../coop-mat-init-length-codegen.cl | 112 ++++++++++++++++++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 94 +++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
new file mode 100644
index 00000000000000..b18f5617338c8f
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -0,0 +1,112 @@
+// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+// ---------------------------------------------------------------------------
+// Type definitions.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init — float scalar.
+// Expected IR: call __spirv_CompositeConstruct with the scalar value,
+// returning a spirv.CooperativeMatrixKHR TargetExtType.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_float
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float
+
+kernel void test_init_float(global float *out) {
+ MatA_t a;
+ a = coop_mat_init(1.0f);
+ (void)a;
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init — integer scalar.
+// Expected IR: call __spirv_CompositeConstruct with an i32 value.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_int
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(i32
+
+kernel void test_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+ (void)m;
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init — zero initialisation.
+// Expected IR: call __spirv_CompositeConstruct with 0.0.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_zero
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+
+kernel void test_init_zero(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ (void)acc;
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length — returns unsigned int.
+// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
+// an i32 alloca.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+
+kernel void test_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ *out = len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_init then coop_mat_length — combined flow.
+// Verifies the TargetExtType produced by init flows into length unchanged.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_then_length
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_init_then_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(2.0f);
+ *out = coop_mat_length(a);
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// The call must NOT be a plain 'call' — it must be 'call spir_func'.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length_cc
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_length_cc(global unsigned int *out) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ *out = coop_mat_length(acc);
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
new file mode 100644
index 00000000000000..ab499bc1b4af7e
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -0,0 +1,94 @@
+// Tests for coop_mat_init and coop_mat_length — Sema (positive).
+//
+// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
+// so arg-count and arg-type validation are not performed by Sema for these two
+// builtins. Only the positive (valid-code) path and the one assignment-target
+// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
+// tested here.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics (sections 1-6 must compile cleanly)
+
+// ---------------------------------------------------------------------------
+// Type definitions shared across all tests.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_B))) MatB_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init: initialise each use role with a float scalar.
+// Two-step (declare then assign) is required because the return type is
+// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_basic(void) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_init(1.0f);
+ b = coop_mat_init(2.0f);
+ c = coop_mat_init(0.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init: integer element type.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_chained(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ acc = coop_mat_init(1.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length: basic usage — returns unsigned int directly.
+// No two-step required for the length call itself.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_basic(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ (void)len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_length: all three use roles.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_roles(void) {
+ MatA_t a; a = coop_mat_init(0.0f);
+ MatB_t b; b = coop_mat_init(0.0f);
+ MatC_t c; c = coop_mat_init(0.0f);
+ unsigned int la = coop_mat_length(a);
+ unsigned int lb = coop_mat_length(b);
+ unsigned int lc = coop_mat_length(c);
+ (void)la; (void)lb; (void)lc;
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length: result used in arithmetic.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_arith(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int half_len = coop_mat_length(a) / 2u;
+ (void)half_len;
+}
>From 569e278b8293d3055066eb80ffe56fa11102e98e Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 19:21:14 -0400
Subject: [PATCH 09/48] Add mangling support
Signed-off-by: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
---
clang/lib/CodeGen/CGBuiltin.cpp | 91 ++++++++-
.../coop-mat-codegen-mangling.cl | 174 ++++++++++++++++++
.../coop-mat-init-length-codegen.cl | 14 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 15 ++
4 files changed, 280 insertions(+), 14 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index da615ae22d45dd..94f2bb2cd3aa97 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2760,6 +2760,62 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
return RetType;
}
+/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
+/// TargetExtType: element type + scope + rows + cols + use.
+/// E.g. target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) ->
+/// "_f32_sc3_16x16_u2"
+static std::string getCoopMatFullSuffix(llvm::Type *Ty) {
+ auto *TET = cast<llvm::TargetExtType>(Ty);
+ llvm::Type *ElemTy = TET->getTypeParameter(0);
+
+ // Element type string
+ std::string ElemStr;
+ if (ElemTy->isFloatTy())
+ ElemStr = "f32";
+ else if (ElemTy->isHalfTy())
+ ElemStr = "f16";
+ else if (ElemTy->isDoubleTy())
+ ElemStr = "f64";
+ else if (auto *IntTy = dyn_cast<llvm::IntegerType>(ElemTy))
+ ElemStr = "i" + std::to_string(IntTy->getBitWidth());
+ else
+ llvm_unreachable("Unsupported cooperative matrix element type");
+
+ // Integer parameters: scope, rows, cols, use
+ unsigned Scope = TET->getIntParameter(0);
+ unsigned Rows = TET->getIntParameter(1);
+ unsigned Cols = TET->getIntParameter(2);
+ unsigned Use = TET->getIntParameter(3);
+
+ return "_" + ElemStr + "_sc" + std::to_string(Scope) + "_" +
+ std::to_string(Rows) + "x" + std::to_string(Cols) + "_u" +
+ std::to_string(Use);
+}
+
+/// Returns a name suffix encoding the address space of a pointer type,
+/// matching OpenCL/SPIR-V address space conventions.
+/// E.g. ptr addrspace(1) -> "_global"
+/// ptr addrspace(3) -> "_local"
+/// ptr addrspace(0) -> "_private"
+/// ptr addrspace(4) -> "_generic"
+static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
+ unsigned AS = cast<llvm::PointerType>(PtrTy)->getAddressSpace();
+ switch (AS) {
+ case 0:
+ return "_private";
+ case 1:
+ return "_global";
+ case 2:
+ return "_constant";
+ case 3:
+ return "_local";
+ case 4:
+ return "_generic";
+ default:
+ return "_as" + std::to_string(AS);
+ }
+}
+
} // namespace
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
@@ -4644,7 +4700,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy = llvm::FunctionType::get(
RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee LoadFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4677,7 +4736,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(1)->getType()));
llvm::FunctionCallee StoreFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4719,7 +4781,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(Arg0->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg1->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg2->getType()));
llvm::FunctionCallee MatMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
@@ -4759,6 +4825,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
(ElTy->isIntegerTy())
? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
: getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ // Function name mangling.
+ Name = Name + getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee BinaryFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
@@ -4788,7 +4856,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
@@ -4816,7 +4886,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarNegFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
@@ -4833,7 +4905,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee InitFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
@@ -4850,7 +4924,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(0)->getType()));
llvm::FunctionCallee LengthFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
new file mode 100644
index 00000000000000..9fec09287f890a
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -0,0 +1,174 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+//
+// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
+// Covers three mangling axes:
+// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
+// (B) coop_mat_mulAdd -- different element types (float vs int)
+// (C) coop_mat_load -- different pointer address spaces
+// (D) coop_mat_store -- different pointer address spaces
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
+// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
+// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+// ---------------------------------------------------------------------------
+// Matrix type aliases used across tests
+// ---------------------------------------------------------------------------
+
+// --- float 16x16 (MulAdd group 1) ---
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatF_16x16_A;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatF_16x16_B;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatF_16x16_C;
+
+// --- float 8x32 / 32x8 / 8x8 (MulAdd group 2 -- same elem type, diff dims) ---
+typedef float __attribute__((coop_mat(SCOPE, 8, 32, USE_A))) MatF_8x32_A;
+typedef float __attribute__((coop_mat(SCOPE, 32, 8, USE_B))) MatF_32x8_B;
+typedef float __attribute__((coop_mat(SCOPE, 8, 8, USE_C))) MatF_8x8_C;
+
+// --- int 16x16 (MulAdd group 3 -- different element type to group 1) ---
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatI_16x16_A;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatI_16x16_B;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatI_16x16_C;
+
+// ===========================================================================
+// (A) MulAdd: same element type (float), different matrix dimensions
+// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
+// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
+// ===========================================================================
+
+kernel void test_muladd_f32_16x16(__global float *ptr) {
+ MatF_16x16_A a;
+ MatF_16x16_B b;
+ MatF_16x16_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+kernel void test_muladd_f32_8x32(__global float *ptr) {
+ MatF_8x32_A a;
+ MatF_32x8_B b;
+ MatF_8x8_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 8);
+ b = coop_mat_load(ptr, ROW_MAJOR, 32);
+ c = coop_mat_load(ptr, ROW_MAJOR, 8);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_8x32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x32_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_32x8_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x8_u{{[0-9]+}}
+
+// ===========================================================================
+// (B) MulAdd: different element types -- float 16x16 vs int 16x16
+// Same dimensions, different elem type -> must produce two distinct symbols.
+// ===========================================================================
+
+kernel void test_muladd_i32_16x16(__global int *iptr) {
+ MatI_16x16_A a;
+ MatI_16x16_B b;
+ MatI_16x16_C c, r;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16);
+ b = coop_mat_load(iptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_i32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _i32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+// ===========================================================================
+// (C) Load: different pointer address spaces
+// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
+// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
+// ===========================================================================
+
+kernel void test_load_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_load_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (D) Store: different pointer address spaces
+// Same matrix type, __global vs __local -> two distinct symbols.
+// ===========================================================================
+
+kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (E) Negative: confirm NO cross-contamination between address space variants
+// i.e. the _global load function is NOT called for the _local case.
+// ===========================================================================
+
+kernel void test_no_cross_contamination(__global float *gptr,
+ __local float *lptr) {
+ MatF_16x16_A ag, al;
+ ag = coop_mat_load(gptr, ROW_MAJOR, 16);
+ al = coop_mat_load(lptr, ROW_MAJOR, 16);
+ (void)ag; (void)al;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_no_cross_contamination
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_global_f32_{{.*}}addrspace(3)
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
+
+// Verify distinct declare lines exist (these appear at end of IR).
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index b18f5617338c8f..8dd1aab2b2a139 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -26,7 +26,7 @@ typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
// CHECK-LABEL: @{{.*}}test_init_float
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_float(global float *out) {
MatA_t a;
@@ -41,7 +41,7 @@ kernel void test_init_float(global float *out) {
// CHECK-LABEL: @{{.*}}test_init_int
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(i32
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_int(void) {
MatA_int_t m;
@@ -56,7 +56,7 @@ kernel void test_init_int(void) {
// CHECK-LABEL: @{{.*}}test_init_zero
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+// CHECK-SAME: @__spirv_CompositeConstruct_{{.*}}(float {{.*}}0
kernel void test_init_zero(void) {
MatC_t acc;
@@ -71,7 +71,7 @@ kernel void test_init_zero(void) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
kernel void test_length(global unsigned int *out) {
@@ -88,8 +88,8 @@ kernel void test_length(global unsigned int *out) {
// CHECK-LABEL: @{{.*}}test_init_then_length
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: @__spirv_CompositeConstruct
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_init_then_length(global unsigned int *out) {
MatA_t a;
@@ -103,7 +103,7 @@ kernel void test_init_then_length(global unsigned int *out) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length_cc
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_length_cc(global unsigned int *out) {
MatC_t acc;
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 7008024ebb3148..7e629e080b8573 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -559,6 +559,21 @@ static bool isNonMangledOCLBuiltin(StringRef Name) {
}
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
+ // Cooperative-matrix builtins emitted by Clang CodeGen carry mangling
+ // suffixes to avoid IR function-signature collisions, e.g.:
+ // "__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global"
+ // Strip everything from the first '_' after "__spirv_CooperativeMatrix"
+ // so the name matches the canonical DemangledNativeBuiltin entry in
+ // SPIRVBuiltins.td (e.g. "__spirv_CooperativeMatrixLoadKHR").
+ static constexpr StringRef CoopMatPrefix = "__spirv_CooperativeMatrix";
+ if (Name.starts_with(CoopMatPrefix)) {
+ size_t SuffixPos = Name.find('_', CoopMatPrefix.size());
+ if (SuffixPos != StringRef::npos)
+ return Name.substr(0, SuffixPos).str();
+ // No mangling suffix present — return the name as-is.
+ return Name.str();
+ }
+
bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
bool IsNonMangledSPIRV = Name.starts_with("__spirv_");
bool IsNonMangledHLSL = Name.starts_with("__hlsl_");
>From fa266049c03d79f34211d333f13d20301e77f6a6 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 13:23:45 -0400
Subject: [PATCH 10/48] Change name of extension to cl_khr_cooperative_matrix
---
clang/include/clang/Basic/Builtins.td | 2 +-
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
.../test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../CodeGenOpenCL/coop-mat-init-length-codegen.cl | 2 +-
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 12 ++++++------
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 8 ++++----
clang/test/SemaOpenCL/coop-mat-init-length-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 6 +++---
11 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 067c112c23ff71..0c53f9979fdcce 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,7 +1931,7 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
-// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+// Cooperative Matrix builtins for cl_khr_cooperative_matrix
def CoopMatLoad : Builtin {
let Spellings = ["coop_mat_load"];
let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 89e96dca85a59d..8a583ad504c706 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9fec09287f890a..9d55bd8c61cc3c 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -8,7 +8,7 @@
// (D) coop_mat_store -- different pointer address spaces
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 58b7a882fdc870..ad8df2939269f5 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -4,7 +4,7 @@
// SPIR-V intrinsic names.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 8dd1aab2b2a139..dfa763b060e720 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -1,7 +1,7 @@
// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 7181efbe7e2a88..f77fe542fd210f 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,6 +1,6 @@
// clang/test/Preprocessor/coop_mat_opencl_ext.cl
//
-// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// Patch 2: cl_khr_cooperative_matrix registration in
// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
//
// Tests: extension macro is predefined when enabled, extension can be
@@ -9,23 +9,23 @@
// ── 2a. Extension macro is predefined when the extension is enabled ─────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
// RUN: | FileCheck %s --check-prefix=EXT
-// EXT: cl_ext_kernel_cooperative_matrix
+// EXT: cl_khr_cooperative_matrix
// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
// RUN: | FileCheck %s --check-prefix=NOEXT
-// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+// NOEXT-NOT: cl_khr_cooperative_matrix
// ── 2c. Enum constants are visible when extension is enabled ─────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index c6230eb94531f4..e8fad815806ece 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -6,24 +6,24 @@
//
// ── 3a. Itanium mangling ────────────────────────────────────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch \
// RUN: -ast-dump %t.pch_probe.cl \
// RUN: | FileCheck %s --check-prefix=PCH
//
// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index ab499bc1b4af7e..799f7a9e15a352 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -7,7 +7,7 @@
// tested here.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics (sections 1-6 must compile cleanly)
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index f8080448b6df51..0c27978d492a92 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -3,7 +3,7 @@
// Patch 4: Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index cec1f278b0037a..bcedd94fdbf17d 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -5,7 +5,7 @@
//
// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 2e6700973e6113..43adeebd945006 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -8,16 +8,16 @@
// TypeLoc operand slots, sizeof / getTypeInfoImpl.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -ast-dump %s \
// RUN: | FileCheck %s --check-prefix=AST
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
>From 49a1cfdae837979f1f54b4da3894b8d106e1ebcc Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 14:37:44 -0400
Subject: [PATCH 11/48] Fix test fails
---
clang/lib/Sema/SemaType.cpp | 6 ++++++
clang/test/AST/undocumented-attrs.cpp | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 36619f92c82964..1bb041e5b9d2fd 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9051,6 +9051,12 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Scope, Use, /* IsCoopMat */ true);
if (!T.isNull())
CurType = T;
+ } else {
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
}
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index 525466b1efdc0c..dff9d3845062a2 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,6 +27,7 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
+CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -88,4 +89,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 82
+CHECK-NEXT: Total: 83
>From 75497cd298d8795253e021ee7ce52a658f6b9b13 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sun, 9 Aug 2026 00:57:18 -0400
Subject: [PATCH 12/48] Fix failing lldb build and a couple of tests
---
.../test/Misc/pragma-attribute-supported-attributes-list.test | 1 +
lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 3 +++
llvm/test/Verifier/coop-mat-verifier-fix.ll | 4 ++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 8f8c1aa3d3f9cd..c906921cc1b7a4 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -63,6 +63,7 @@
// CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record)
// CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record)
// CHECK-NEXT: Convergent (SubjectMatchRule_function)
+// CHECK-NEXT: CoopMatrixType (SubjectMatchRule_type_alias, SubjectMatchRule_variable, SubjectMatchRule_variable_is_parameter, SubjectMatchRule_field)
// CHECK-NEXT: CoroAwaitElidable (SubjectMatchRule_record)
// CHECK-NEXT: CoroAwaitElidableArgument (SubjectMatchRule_variable_is_parameter)
// CHECK-NEXT: CoroDisableLifetimeBound (SubjectMatchRule_function)
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index ee7415354bb479..a8971946198c09 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -4306,6 +4306,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
// Matrix types that we're not sure how to display at the moment.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5183,6 +5184,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type) {
break;
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5356,6 +5358,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
// Matrix types we're not sure how to display yet.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
index c4e206b76b7603..7f2eedb657d41d 100644
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -22,14 +22,14 @@
declare void @llvm.experimental.noalias.scope.decl(metadata)
define void @test_no_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
; Third arg present and <= 64 bits wide — must also pass
define void @test_valid_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
>From 69e02ba4be5f021f2fa60ec96f08f83b37109bfc Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 4 Sep 2026 18:58:24 -0400
Subject: [PATCH 13/48] Minor changes to sync with latest upstream changes
---
clang/lib/AST/ASTContext.cpp | 10 ++++------
clang/lib/CodeGen/CGBuiltin.cpp | 2 --
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 93fc3b3da69ff3..70c0c97aaf6d89 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4972,9 +4972,8 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
- void *InsertPos = nullptr;
- if (CooperativeMatrixType *MTP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ llvm::FoldingSetInsertToken Token;
+ if (CooperativeMatrixType *MTP = CooperativeMatrixTypes.lookup(ID, Token))
return QualType(MTP, 0);
QualType Canonical;
@@ -4982,15 +4981,14 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
NumRows, NumColumns, Use);
- CooperativeMatrixType *NewIP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ CooperativeMatrixType *NewIP = CooperativeMatrixTypes.lookup(ID, Token);
assert(!NewIP && "Matrix type shouldn't already exist in the map");
(void)NewIP;
}
auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
- CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ CooperativeMatrixTypes.insert(New, Token);
Types.push_back(New);
return QualType(New, 0);
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 94f2bb2cd3aa97..5c39dcad6d58d2 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2816,8 +2816,6 @@ static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
}
}
-} // namespace
-
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
>From f706f3920909a0793e876fd6246313acf68bbfc7 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 13:55:14 -0400
Subject: [PATCH 14/48] Update CooperativeMatrixType class definition to be
derived directly from Type class, instead of MatrixType class.
---
clang/include/clang/AST/OperationKinds.def | 3 +
clang/include/clang/AST/TypeBase.h | 78 ++++++---
clang/include/clang/AST/TypeLoc.h | 105 +++++++----
clang/include/clang/AST/TypeProperties.td | 3 +
clang/include/clang/Basic/TypeNodes.td | 2 +-
clang/include/clang/CIR/Dialect/IR/CIROps.td | 1 +
clang/include/clang/Sema/Sema.h | 27 ++-
clang/lib/AST/ASTContext.cpp | 18 +-
clang/lib/AST/Expr.cpp | 1 +
clang/lib/AST/ExprConstant.cpp | 2 +
clang/lib/AST/Type.cpp | 16 --
clang/lib/CIR/CodeGen/CIRGenExpr.cpp | 2 +
clang/lib/CodeGen/CGExpr.cpp | 78 +++++----
clang/lib/CodeGen/CGExprAgg.cpp | 2 +
clang/lib/CodeGen/CGExprComplex.cpp | 1 +
clang/lib/CodeGen/CGExprConstant.cpp | 1 +
clang/lib/CodeGen/CGExprScalar.cpp | 16 +-
clang/lib/CodeGen/CodeGenFunction.h | 23 ++-
clang/lib/CodeGen/CodeGenTBAA.cpp | 5 +-
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +-
clang/lib/CodeGen/QualTypeMapper.h | 1 -
clang/lib/Edit/RewriteObjCFoundationAPI.cpp | 1 +
clang/lib/Sema/SemaCast.cpp | 16 ++
clang/lib/Sema/SemaChecking.cpp | 32 ++--
clang/lib/Sema/SemaDecl.cpp | 2 +-
clang/lib/Sema/SemaExpr.cpp | 175 ++++++++++++++-----
clang/lib/Sema/SemaTemplateDeduction.cpp | 8 +-
clang/lib/Sema/SemaType.cpp | 145 ++++++++++-----
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 26 +--
clang/tools/libclang/CIndex.cpp | 2 +-
30 files changed, 521 insertions(+), 277 deletions(-)
diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def
index 8a13ad988403bf..d0cda0551f2d52 100644
--- a/clang/include/clang/AST/OperationKinds.def
+++ b/clang/include/clang/AST/OperationKinds.def
@@ -185,6 +185,9 @@ CAST_OPERATION(ToVoid)
/// CK_MatrixCast - A cast between matrix types of the same dimensions.
CAST_OPERATION(MatrixCast)
+/// CK_CoopMatrixCast - A cast between compatible cooperative matrix types.
+CAST_OPERATION(CoopMatrixCast)
+
/// CK_VectorSplat - A conversion from an arithmetic type to a
/// vector of that element type. Fills all elements ("splats") with
/// the source value.
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index b8146904612239..0b83ac3fd03e40 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4483,7 +4483,6 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
- T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4579,65 +4578,81 @@ class ConstantMatrixType final : public MatrixType {
}
};
-/// Represents a cooperative matrix type.
-class CooperativeMatrixType final : public MatrixType {
+/// Represents an opaque OpenCL cooperative matrix type.
+///
+/// Unlike MatrixType, a cooperative matrix is not an ordinary matrix value.
+/// It is an opaque, distributed object whose shape and role are part of its
+/// type identity.
+class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
protected:
friend class ASTContext;
+ /// Element type of the cooperative matrix.
+ QualType ElementType;
+
/// Number of rows and columns.
unsigned NumRows;
unsigned NumColumns;
- /// Scope and use
+ /// Cooperative matrix scope and use.
unsigned Scope;
unsigned Use;
static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
- CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
- unsigned NRows, unsigned NColumns, unsigned Use,
- QualType CanonElementType);
+ CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
+ unsigned NumColumns, unsigned Use,
+ QualType CanonicalType)
+ : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
+ NumRows, NumColumns, Use, CanonicalType) {}
- CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
- unsigned Scope, unsigned NRows, unsigned NColumns,
- unsigned Use, QualType CanonElementType);
+ CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use, QualType CanonicalType)
+ : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
+ Scope(Scope), Use(Use) {}
public:
- /// Returns the number of rows in the matrix.
+ /// Returns the element type.
+ QualType getElementType() const { return ElementType; }
+
+ /// Returns the number of rows.
unsigned getNumRows() const { return NumRows; }
- /// Returns the number of columns in the matrix.
+ /// Returns the number of columns.
unsigned getNumColumns() const { return NumColumns; }
- /// Returns the scope of the matrix.
+ /// Returns the cooperative matrix scope.
unsigned getScope() const { return Scope; }
- /// Returns the use of the matrix.
+ /// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into a vector.
+ /// Returns the number of elements required to embed the matrix into
+ /// a vector representation.
unsigned getNumElementsFlattened() const {
return getNumRows() * getNumColumns();
}
- /// Returns true if \p NumElements is a valid matrix dimension.
+ /// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
}
- /// Return true if \p Scope is valid
+ /// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
+ return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
}
- /// Return true if \p Use is valid
+ /// Returns true if \p Use is a valid cooperative matrix use.
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
+ return Use == 0 || // CLK_COOPERATIVE_MATRIX_A
+ Use == 1 || // CLK_COOPERATIVE_MATRIX_B
+ Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum number of elements per dimension.
+ /// Returns the maximum valid number of elements per dimension.
static constexpr unsigned getMaxElementsPerDimension() {
return MaxElementsPerDimension;
}
@@ -4658,6 +4673,23 @@ class CooperativeMatrixType final : public MatrixType {
ID.AddInteger(TypeClass);
}
+ static bool isValidElementType(QualType ElemTy) {
+ return ElemTy->isSpecificBuiltinType(BuiltinType::Char_S) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UChar) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Short) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UShort) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Int) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UInt) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Long) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::ULong) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Half) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Float) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Double);
+ }
+
+ bool isSugared() const { return false; }
+ QualType desugar() const { return QualType(this, 0); }
+
static bool classof(const Type *T) {
return T->getTypeClass() == CooperativeMatrix;
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 24fcda8f4c7e58..122090f7e6ec57 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,10 +2142,8 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
- Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
- Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2153,50 +2151,26 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
+ /// ^~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
- void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
-
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
- void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
-
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2213,10 +2187,8 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
- setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
- setAttrUseOperand(nullptr);
}
};
@@ -2224,15 +2196,82 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
-class CooperativeMatrixTypeLoc
- : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
- CooperativeMatrixType> {};
-
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
DependentSizedMatrixType> {};
+struct CooperativeMatrixTypeLocInfo {
+ SourceLocation AttrLoc;
+ SourceRange OperandParens;
+ Expr *ScopeOperand;
+ Expr *RowOperand;
+ Expr *ColumnOperand;
+ Expr *UseOperand;
+};
+
+class CooperativeMatrixTypeLoc
+ : public ConcreteTypeLoc<UnqualTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType,
+ CooperativeMatrixTypeLocInfo> {
+public:
+ /// The location of the attribute name, i.e.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^^~~~~~~~~~~~~~~~~~~
+ SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
+ void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+
+ /// The attribute's scope operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
+ /// The attribute's row operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
+ void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
+
+ /// The attribute's column operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
+ void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+
+ /// The attribute's use operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
+ /// The location of the parentheses around the operand, if there is
+ /// an operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^
+ SourceRange getAttrOperandParensRange() const {
+ return getLocalData()->OperandParens;
+ }
+ void setAttrOperandParensRange(SourceRange range) {
+ getLocalData()->OperandParens = range;
+ }
+
+ SourceRange getLocalSourceRange() const {
+ SourceRange range(getAttrNameLoc());
+ range.setEnd(getAttrOperandParensRange().getEnd());
+ return range;
+ }
+
+ void initializeLocal(ASTContext &Context, SourceLocation loc) {
+ setAttrNameLoc(loc);
+ setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
+ setAttrRowOperand(nullptr);
+ setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
+ }
+};
+
// FIXME: location of the '_Complex' keyword.
class ComplexTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
ComplexTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 88d84d6d3234f5..d134edc39e92ce 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -264,6 +264,9 @@ let Class = ConstantMatrixType in {
}
let Class = CooperativeMatrixType in {
+ def : Property<"elementType", QualType> {
+ let Read = [{ node->getElementType() }];
+ }
def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index e6f396245070c6..28e4b94c3bea92 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,7 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
-def CooperativeMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<Type>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 63d50761806a67..7d8bc67e7bb41e 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -233,6 +233,7 @@ def CIR_CastKind : CIR_I32Enum<"CastKind", "cast kind", [
I32EnumCase<"ptr_to_bool", 23>,
// CK_ToVoid
// CK_MatrixCast
+ // CK_CoopMatrixCast
// CK_VectorSplat
I32EnumCase<"integral", 27>,
I32EnumCase<"int_to_bool", 28>,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 61ccd3ad7f2185..95ba15f9e0077c 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8031,6 +8031,14 @@ class Sema final : public SemaBase {
QualType CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
SourceLocation Loc, bool IsCompAssign);
+ /// Type checking for cooperative matrix binary operators.
+ QualType CheckCoopMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+ QualType CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+
/// Are the two types SVE-bitcast-compatible types? I.e. is bitcasting from
/// the first SVE type (e.g. an SVE VLAT) to the second type (e.g. an SVE
/// VLST) allowed?
@@ -8043,6 +8051,11 @@ class Sema final : public SemaBase {
/// do they have the same number of rows and the same number of columns?
bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ /// Are the two types cooperative matrix types and do they have the same
+ /// dimensions i.e. do they have the same number of rows and the same number
+ /// of columns? Also do they have the same scope and use?
+ bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+
bool areVectorTypesSameSize(QualType srcType, QualType destType);
/// Are the two types lax-compatible vector types? That is, given
@@ -8085,6 +8098,13 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
+ // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
+ // We allow casting between cooperative matrixes of the same scope, use, and
+ // same dimensions i.e. when they have the same number of rows and columns.
+ // Returns true if the cast is invalid.
+ bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind);
+
// CheckVectorCast - check type constraints for vectors.
// Since vectors are an extension, there are no C standard reference for this.
// We allow casting between vectors and integer datatypes of the same size.
@@ -15414,8 +15434,11 @@ class Sema final : public SemaBase {
SourceLocation AttrLoc);
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc, Expr *Scope = nullptr,
- Expr *Use = nullptr, bool IsCoopMat = false);
+ SourceLocation AttrLoc);
+
+ QualType BuildCoopMatrixType(QualType T, Expr *Scope, Expr *Use,
+ Expr *NumRows, Expr *NumColumns,
+ SourceLocation AttrLoc);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 70c0c97aaf6d89..926e7269450a12 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2222,9 +2222,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
case Type::CooperativeMatrix: {
const auto *MT = cast<CooperativeMatrixType>(T);
TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
- // The internal layout of a matrix value is implementation defined.
- // Initially be ABI compatible with arrays with respect to alignment and
- // size.
+ // The internal layout of a cooperative matrix value is implementation
+ // defined. Initially be ABI compatible with arrays with respect to
+ // alignment and size.
Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
Align = ElementInfo.Align;
break;
@@ -10830,10 +10830,10 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatMatrixTypes - Return true if the two specified matrix types are
-/// compatible.
-static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
- const CooperativeMatrixType *RHS) {
+/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
+/// matrix types are compatible.
+static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
return LHS->getElementType() == RHS->getElementType() &&
LHS->getScope() == RHS->getScope() &&
@@ -12357,8 +12357,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
return LHS;
return {};
case Type::CooperativeMatrix:
- if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
- RHSCan->castAs<CooperativeMatrixType>()))
+ if (areCompatCoopMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
return LHS;
return {};
case Type::ObjCObject: {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 1ba2ccdf697f9d..e717c053b8ed42 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1953,6 +1953,7 @@ bool CastExpr::CastConsistency() const {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
assert(!getType()->isBooleanType() && "unheralded conversion to bool");
goto CheckNoBasePath;
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 554839160228fc..44990fd15bc9b8 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -20217,6 +20217,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointCast:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLAggregateSplatCast:
llvm_unreachable("invalid cast kind for integral value");
@@ -21149,6 +21150,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLElementwiseCast:
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 020564f3edfb79..1c4a648003846e 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -510,20 +510,6 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns), Layout(Layout) {}
-CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
- nColumns, use, canonType) {}
-
-CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : MatrixType(tc, matrixType, canonType), NumRows(nRows),
- NumColumns(nColumns), Scope(scope), Use(use) {}
-
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -3304,8 +3290,6 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
- if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
- return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index 9072a22cef52bd..9e2d1f369c0613 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -261,6 +261,7 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
case CK_IntegralToFloating:
case CK_LValueBitCast:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_MemberPointerToBoolean:
case CK_NonAtomicToAtomic:
case CK_ObjCObjectLValueCast:
@@ -1727,6 +1728,7 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index c8363095fc5457..ec0f2662d26ddb 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7655,58 +7655,68 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy) {
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+llvm::Value *
+CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy) {
+ assert(!Args.empty() &&
+ "Expected at least one argument for cooperative matrix builtin");
+
+ llvm::LLVMContext &Context = CGM.getLLVMContext();
+ llvm::Module &Module = CGM.getModule();
+
+ llvm::Type *ResultLLVMType = ConvertType(ResultTy);
+
+ SmallVector<llvm::Type *, 8> ArgTypes;
+ ArgTypes.reserve(Args.size());
+
+ for (llvm::Value *Arg : Args) {
+ assert(Arg && "Cooperative matrix builtin argument must not be null");
+ ArgTypes.push_back(Arg->getType());
+ }
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+ // Although the frontend builtin is declared as void(...), the LLVM-level
+ // builtin returns the cooperative matrix result directly.
+ llvm::FunctionType *BuiltinType =
+ llvm::FunctionType::get(ResultLLVMType, ArgTypes,
+ /*IsVarArg=*/false);
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
- "coopmat.broadcast");
+ llvm::FunctionCallee Builtin =
+ Module.getOrInsertFunction(BuiltinName, BuiltinType);
+
+ return Builder.CreateCall(Builtin, Args, "coopmat.result");
}
llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
llvm::Value *LHS,
llvm::Value *RHS,
QualType ResultTy) {
- auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
-
switch (Opcode) {
case BO_Add:
- return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_add", {LHS, RHS}, ResultTy);
case BO_Sub:
- return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub(LHS, RHS, "coopmat.isub");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_sub", {LHS, RHS}, ResultTy);
case BO_Mul:
- // mat * mat → element-wise FMul/IMul
- // mat * scalar → handled separately (see scalar path below)
+ // Matrix * scalar.
if (!RHS->getType()->isVectorTy()) {
- // mat * scalar: broadcast scalar then element-wise mul
- llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
- return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_mul", {LHS, RHS},
+ ResultTy);
}
- return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul(LHS, RHS, "coopmat.imul");
+
+ // Matrix * matrix.
+ return EmitCoopMatBuiltinCall("coop_mat_binary_mul", {LHS, RHS}, ResultTy);
case BO_Div:
- if (IsFloat)
- return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned)
- return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
- return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_div", {LHS, RHS}, ResultTy);
default:
- llvm_unreachable("Unsupported cooperative matrix binary op");
+ llvm_unreachable("Unsupported cooperative matrix binary operation");
}
-}
\ No newline at end of file
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatNeg(llvm::Value *Operand,
+ QualType ResultTy) {
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_neg", {Operand}, ResultTy);
+}
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 73c6301c5d585b..0ec1877f665092 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1091,6 +1091,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
case CK_BuiltinFnToFnPtr:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_IntToOCLSampler:
@@ -1547,6 +1548,7 @@ static bool castPreservesZero(const CastExpr *CE) {
// Language extensions.
case CK_VectorSplat:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_NonAtomicToAtomic:
case CK_AtomicToNonAtomic:
case CK_HLSLVectorTruncation:
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 30d693fb371cb6..87877bed010c14 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -618,6 +618,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index e2ec07bbae711a..5aab1dfd6ac0c4 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1388,6 +1388,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index ac37f3da397ffa..20b249bf032d9f 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -3018,7 +3018,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
CGF.EmitIgnoredExpr(E);
return nullptr;
}
- case CK_MatrixCast: {
+ case CK_MatrixCast:
+ case CK_CoopMatrixCast: {
return EmitScalarConversion(Visit(E), E->getType(), DestTy,
CE->getExprLoc());
}
@@ -3692,15 +3693,10 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
- if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
- llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy =
- E->getType()->getAs<CooperativeMatrixType>()->getElementType();
- if (CompTy->isFloatingType())
- return Builder.CreateFNeg(Val, "coopmat.fneg");
- else
- return Builder.CreateNeg(Val, "coopmat.ineg");
- }
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType())
+ return CGF.EmitCoopMatNeg(CGF.EmitScalarExpr(E->getSubExpr()),
+ E->getType());
+
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6604a46f43dccd..03380c36a7eb89 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5192,15 +5192,6 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
- /// Helper function for EmitCoopMatBinaryOp
- llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy);
-
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
- llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
- llvm::Value *RHS, QualType ResultTy);
-
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
@@ -5544,6 +5535,20 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
SourceLocation loc);
+ /// Helper function to emit coop matrix builtin call.
+ llvm::Value *EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
+ /// EmitCoopMatNeg - Emit the computation of the negate op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
+
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
/// point operation, expressed as the maximum relative error in ulp.
void SetFPAccuracy(llvm::Value *Val, float Accuracy);
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index c0ba3582076415..540dbc069c62b1 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -332,11 +332,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
+ assert(isa<ConstantMatrixType>(Ty) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
+ if (const auto *CMTy = dyn_cast<CooperativeMatrixType>(Ty))
+ return getTypeInfo(CMTy->getElementType());
+
// Enum types are distinct types. In C++ they have "underlying types",
// however they aren't related for TBAA.
if (const EnumType *ETy = dyn_cast<EnumType>(Ty)) {
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 5850604f88207b..6909370d7f1090 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -137,10 +137,8 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
case Type::CooperativeMatrix: {
- const auto *MT = cast<CooperativeMatrixType>(QT);
- return Builder.getArrayType(convertType(MT->getElementType()),
- MT->getNumRows() * MT->getNumColumns(),
- ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ llvm::reportFatalInternalError(
+ "Cooperative Matrix type not supported in ABI lowering library");
}
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index 73d1753349fd40..f6bb43482c705e 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -48,7 +48,6 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
- const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index e8d4660fd36b22..ecc7805ddd446d 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -1079,6 +1079,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
case CK_ZeroToOCLOpaqueType:
case CK_IntToOCLSampler:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
return false;
case CK_BooleanToSignedIntegral:
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index c797bdb11e0cbd..c472cbeac9ef95 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -1607,6 +1607,15 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
return TC_Success;
}
+ if (SrcType->isCooperativeMatrixType() &&
+ DestType->isCooperativeMatrixType()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind)) {
+ SrcExpr = ExprError();
+ return TC_Failed;
+ }
+ return TC_Success;
+ }
+
if (SrcType == Self.Context.AMDGPUFeaturePredicateTy &&
DestType == Self.Context.getLogicalOperationType()) {
SrcExpr = Self.AMDGPU().ExpandAMDGPUPredicateBuiltIn(SrcExpr.get());
@@ -3247,6 +3256,13 @@ void CastOperation::CheckCStyleCast() {
return;
}
+ if (DestType->getAs<CooperativeMatrixType>() ||
+ SrcType->getAs<CooperativeMatrixType>()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind))
+ SrcExpr = ExprError();
+ return;
+ }
+
if (const VectorType *DestVecTy = DestType->getAs<VectorType>()) {
if (Self.CheckAltivecInitFromScalar(OpRange, DestType, SrcType)) {
SrcExpr = ExprError();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9e8c42975f025a..77c2fbcf4c397d 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17476,7 +17476,7 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
ArgError = true;
} else {
ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
- if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
ArgError = true;
}
}
@@ -17596,7 +17596,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
M2Ty->getElementType().getUnqualifiedType())
Diag(Loc, diag::err_coop_matrix_element_type);
- if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
}
@@ -17611,7 +17611,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17733,9 +17733,8 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
- auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
- if (!ConstMType && !CoopMType) {
+ auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
+ if (!MType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17744,23 +17743,12 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- if (ConstMType) {
- QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
- if (CoopMType) {
- QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(), CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
-
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 232cb88d3eec73..24134fc65be2e5 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14676,7 +14676,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
- if (!VDecl->getType()->isMatrixType()) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c056a224a91256..af5cddd606ce1a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5193,7 +5193,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
}
// If the base is a matrix type, try to create a new MatrixSubscriptExpr.
- if (base->getType()->isMatrixType()) {
+ if (base->getType()->isMatrixType() ||
+ base->getType()->isCooperativeMatrixType()) {
if (CheckAndReportCommaError(ArgExprs.front()))
return ExprError();
@@ -8092,16 +8093,6 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
- if (srcTy->isCooperativeMatrixType()) {
- const CooperativeMatrixType *matSrcType =
- srcTy->getAs<CooperativeMatrixType>();
- const CooperativeMatrixType *matDestType =
- destTy->getAs<CooperativeMatrixType>();
-
- return matSrcType->getNumRows() == matDestType->getNumRows() &&
- matSrcType->getNumColumns() == matDestType->getNumColumns();
- }
-
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -8109,6 +8100,22 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
+bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
+ QualType destTy) {
+ if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
+ return false;
+
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getScope() == matDestType->getScope() &&
+ matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns() &&
+ matSrcType->getUse() == matDestType->getUse();
+}
+
bool Sema::areVectorTypesSameSize(QualType SrcTy, QualType DestTy) {
assert(DestTy->isVectorType() || SrcTy->isVectorType());
@@ -8222,6 +8229,27 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
return false;
}
+bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind) {
+ if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
+ if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
+ << DestTy << SrcTy << R;
+ }
+ } else if (SrcTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << SrcTy << DestTy << R;
+ } else if (DestTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << DestTy << SrcTy << R;
+ }
+
+ Kind = CK_CoopMatrixCast;
+ return false;
+}
+
bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
CastKind &Kind) {
assert(VectorTy->isVectorType() && "Not a vector type!");
@@ -11451,6 +11479,9 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
ArithConvKind::Arithmetic);
if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
+ if (!IsDiv &&
+ (LHSTy->isCooperativeMatrixType() || RHSTy->isCooperativeMatrixType()))
+ return CheckCoopMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
@@ -11859,7 +11890,7 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -12014,7 +12045,7 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -14067,9 +14098,10 @@ QualType Sema::CheckMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
return InvalidOperands(Loc, LHS, RHS);
}
-QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
- SourceLocation Loc,
- bool IsCompAssign) {
+QualType Sema::CheckCoopMatrixElementwiseOperands(ExprResult &LHS,
+ ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
if (!IsCompAssign) {
LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
if (LHS.isInvalid())
@@ -14079,33 +14111,55 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
- if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
- auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
- auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) &&
- "At least one operand must be a matrix");
- if (LHSMatType && RHSMatType) {
- if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
- return InvalidOperands(Loc, LHS, RHS);
+ // For conversion purposes, we ignore any qualifiers.
+ // For example, "const float" and "float" are equivalent.
+ QualType LHSType = LHS.get()->getType().getUnqualifiedType();
+ QualType RHSType = RHS.get()->getType().getUnqualifiedType();
- if (Context.hasSameType(LHSMatType, RHSMatType))
- return Context.getCommonSugaredType(
- LHS.get()->getType().getUnqualifiedType(),
- RHS.get()->getType().getUnqualifiedType());
+ const CooperativeMatrixType *LHSMatType =
+ LHSType->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *RHSMatType =
+ RHSType->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a cooperative matrix");
- QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
- if (!Context.hasSameType(LHSELTy, RHSELTy))
- return InvalidOperands(Loc, LHS, RHS);
+ if (Context.hasSameType(LHSType, RHSType))
+ return Context.getCommonSugaredType(LHSType, RHSType);
- return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy),
- LHSMatType->getScope(), LHSMatType->getNumRows(),
- RHSMatType->getNumColumns(), LHSMatType->getUse());
- }
+ // Type conversion may change LHS/RHS. Keep copies to the original results, in
+ // case we have to return InvalidOperands.
+ ExprResult OriginalLHS = LHS;
+ ExprResult OriginalRHS = RHS;
+ if (LHSMatType && !RHSMatType) {
+ RHS = tryConvertExprToType(RHS.get(), LHSMatType->getElementType());
+ if (!RHS.isInvalid())
+ return LHSType;
+
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
}
+ if (!LHSMatType && RHSMatType) {
+ LHS = tryConvertExprToType(LHS.get(), RHSMatType->getElementType());
+ if (!LHS.isInvalid())
+ return RHSType;
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
+ }
+
+ return InvalidOperands(Loc, LHS, RHS);
+}
+
+QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -14131,6 +14185,43 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
}
+QualType Sema::CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ return CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+}
+
static bool isLegalBoolVectorBinaryOp(BinaryOperatorKind Opc) {
switch (Opc) {
default:
@@ -15909,7 +16000,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
// Cooperative matrix support
- if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ if (LHSTy->isCooperativeMatrixType() && RHSTy->isCooperativeMatrixType()) {
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
@@ -15917,7 +16008,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
@@ -15947,7 +16038,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
- if (!LHSExpr->getType()->isMatrixType()) {
+ if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
@@ -16617,7 +16708,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
- if (Opc == UO_Minus && Ty->isMatrixType()) {
+ if (Opc == UO_Minus && Ty->isCooperativeMatrixType()) {
return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
{InputExpr});
}
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 284561435727e8..96aca99c16a635 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2467,9 +2467,11 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
if (!MA)
return TemplateDeductionResult::NonDeducedMismatch;
- // Check that the dimensions are the same
- if (MP->getNumRows() != MA->getNumRows() ||
- MP->getNumColumns() != MA->getNumColumns()) {
+ // Check that the dimensions, scope and use are the same
+ if (MP->getScope() != MA->getScope() ||
+ MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns() ||
+ MP->getUse() != MA->getUse()) {
return TemplateDeductionResult::NonDeducedMismatch;
}
// Perform deduction on element types.
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 1bb041e5b9d2fd..395e0ae920887e 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2532,11 +2532,9 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc, Expr *ScopeExpr,
- Expr *UseExpr, bool IsCoopMat) {
- if (!IsCoopMat)
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc) {
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2617,29 +2615,76 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
- if (IsCoopMat) {
- std::optional<llvm::APSInt> ValueScope =
- ScopeExpr->getIntegerConstantExpr(Context);
- unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
- std::optional<llvm::APSInt> ValueUse =
- UseExpr->getIntegerConstantExpr(Context);
- unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
-
- if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr)
- << ColRange << "matrix scope";
- return QualType();
- }
- if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
- return QualType();
- }
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
- }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
+QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
+ Expr *NumRows, Expr *NumCols, Expr *UseExpr,
+ SourceLocation AttrLoc) {
+ std::optional<llvm::APSInt> ValueRows =
+ NumRows->getIntegerConstantExpr(Context);
+ std::optional<llvm::APSInt> ValueColumns =
+ NumCols->getIntegerConstantExpr(Context);
+
+ auto const RowRange = NumRows->getSourceRange();
+ auto const ColRange = NumCols->getSourceRange();
+
+ // Both are row and column expressions are invalid.
+ if (!ValueRows && !ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange << ColRange;
+ return QualType();
+ }
+
+ // Only the row expression is invalid.
+ if (!ValueRows) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange;
+ return QualType();
+ }
+
+ // Only the column expression is invalid.
+ if (!ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << ColRange;
+ return QualType();
+ }
+
+ // Check the matrix dimensions.
+ unsigned MatrixRows = static_cast<unsigned>(ValueRows->getZExtValue());
+ unsigned MatrixColumns = static_cast<unsigned>(ValueColumns->getZExtValue());
+ if (MatrixRows == 0 && MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size)
+ << "matrix" << RowRange << ColRange;
+ return QualType();
+ }
+ if (MatrixRows == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << RowRange;
+ return QualType();
+ }
+ if (MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << ColRange;
+ return QualType();
+ }
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+}
+
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
if ((T->isArrayType() && !getLangOpts().allowArrayReturnTypes()) ||
T->isFunctionType()) {
@@ -6014,7 +6059,7 @@ static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
}
}
- llvm_unreachable("no matrix_type attribute found at the expected location!");
+ llvm_unreachable("no coop_mat attribute found at the expected location!");
}
namespace {
@@ -9030,34 +9075,40 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S, bool IsCoopMat = false) {
- if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
+ Sema &S) {
+ if (!S.getLangOpts().MatrixTypes) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- unsigned int NumAttrs = IsCoopMat ? 4 : 2;
- if (Attr.getNumArgs() != NumAttrs) {
+ if (Attr.getNumArgs() != 2) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << NumAttrs;
+ << Attr << 2;
return;
}
- if (IsCoopMat) {
- Expr *Scope = Attr.getArgAsExpr(0);
- Expr *RowsExpr = Attr.getArgAsExpr(1);
- Expr *ColsExpr = Attr.getArgAsExpr(2);
- Expr *Use = Attr.getArgAsExpr(3);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
- Scope, Use, /* IsCoopMat */ true);
- if (!T.isNull())
- CurType = T;
- } else {
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
+}
+
+/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
+ Sema &S) {
+ if (Attr.getNumArgs() != 4) {
+ S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+ << Attr << 4;
+ return;
}
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildCoopMatrixType(CurType, Scope, RowsExpr, ColsExpr, Use,
+ Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9342,7 +9393,7 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
break;
case ParsedAttr::AT_CoopMatrixType:
- HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ HandleCoopMatrixTypeAttr(type, attr, state.getSema());
attr.setUsedAsTypeAttr();
break;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 43adeebd945006..f4c78b965d24aa 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -30,7 +30,7 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
// ---------------------------------------------------------------------------
-// 1. Basic type construction — four use roles, float element type.
+// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
@@ -43,7 +43,7 @@ typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
// AST: TypedefDecl {{.*}} MatA_int8x8
// ---------------------------------------------------------------------------
-// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// Type printer — VarDecls carry the coop_mat attribute in their type string.
// ---------------------------------------------------------------------------
void test_type_spelling(void) {
MatA_float16x16 a;
@@ -58,17 +58,7 @@ void test_type_spelling(void) {
// AST: VarDecl {{.*}} d {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
-// float(4B)*16*16 = 1024 B = 8192 bits
-// int(4B)*8*8 = 256 B = 2048 bits
-// ---------------------------------------------------------------------------
-void test_sizeof(void) {
- _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
- _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
-}
-
-// ---------------------------------------------------------------------------
-// 4. Parameter / return type — type preserved through function boundary.
+// Parameter / return type — type preserved through function boundary.
// ---------------------------------------------------------------------------
MatA_float16x16 test_param_return(MatA_float16x16 in) {
return in;
@@ -78,7 +68,7 @@ MatA_float16x16 test_param_return(MatA_float16x16 in) {
// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 5. TypeLoc operand traversal — all four operand slots populated.
+// TypeLoc operand traversal — all four operand slots populated.
// ---------------------------------------------------------------------------
void test_typeloc_operands(void) {
float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
@@ -88,7 +78,7 @@ void test_typeloc_operands(void) {
// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// RecursiveASTVisitor — element type (half) reachable through the node.
// ---------------------------------------------------------------------------
void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
(void)x;
@@ -97,8 +87,8 @@ void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))
// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
-// same canonical type; taking a pointer across them compiles cleanly.
+// mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
@@ -109,6 +99,6 @@ void test_merge_types(void) {
}
// ---------------------------------------------------------------------------
-// 8. PCH serialisation round-trip.
+// PCH serialisation round-trip.
// ---------------------------------------------------------------------------
// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 951fedea44ee4d..dac8117f971d2a 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1912,7 +1912,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
-DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, Type)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
>From a42c198634fe51705f297e755889face9ff282d1 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 14:31:27 -0400
Subject: [PATCH 15/48] iMinor issues with previous commit
---
clang/include/clang/AST/TypeLoc.h | 2 +-
clang/lib/CodeGen/CGExpr.cpp | 2 +-
clang/lib/Sema/SemaChecking.cpp | 3 +--
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 122090f7e6ec57..b7432d03a92dd7 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2217,7 +2217,7 @@ class CooperativeMatrixTypeLoc
public:
/// The location of the attribute name, i.e.
/// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^^~~~~~~~~~~~~~~~~~~
+ /// ^~~~~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index ec0f2662d26ddb..0bb314d86897e3 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -6377,6 +6377,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
@@ -7662,7 +7663,6 @@ CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
assert(!Args.empty() &&
"Expected at least one argument for cooperative matrix builtin");
- llvm::LLVMContext &Context = CGM.getLLVMContext();
llvm::Module &Module = CGM.getModule();
llvm::Type *ResultLLVMType = ConvertType(ResultTy);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 77c2fbcf4c397d..34baa40ebd3755 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17744,8 +17744,7 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
>From c48391676c18d2879a0e04d5839c74bae320c522 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 15:35:07 -0400
Subject: [PATCH 16/48] Add regression test for change in SPIRVUtils.cpp
---
.../SPIRV/coop-matrix-builtin-demangle.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
diff --git a/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
new file mode 100644
index 00000000000000..e99d2199b7131e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=spirv64-unknown-unknown \
+; RUN: -spirv-ext=+SPV_KHR_cooperative_matrix \
+; RUN: -filetype=asm < %s | FileCheck %s
+
+target triple = "spirv64-unknown-unknown"
+
+%coopmat.f32.sc3.16x16.u0 =
+ type target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+
+; ---------------------------------------------------------------------------
+; Suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_mangled_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_mangled_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
+; ---------------------------------------------------------------------------
+; No-suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_canonical_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_canonical_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
>From dcf33b6062b5be418baa9925fefc03c14c78390d Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 17:00:22 -0400
Subject: [PATCH 17/48] Address review comments
---
clang/include/clang/AST/TypeBase.h | 19 ++------
clang/include/clang/AST/TypeProperties.td | 6 +--
.../include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/lib/AST/ASTStructuralEquivalence.cpp | 2 +-
clang/lib/Headers/opencl-c-base.h | 47 ++++++++++---------
5 files changed, 33 insertions(+), 43 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 0b83ac3fd03e40..5328e2db64d2c4 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4603,13 +4603,8 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
- : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
- NumRows, NumColumns, Use, CanonicalType) {}
-
- CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
- unsigned Scope, unsigned NumRows, unsigned NumColumns,
- unsigned Use, QualType CanonicalType)
- : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ : Type(Type::CooperativeMatrix, CanonicalType,
+ ElementType->getDependence()),
ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
Scope(Scope), Use(Use) {}
@@ -4629,12 +4624,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into
- /// a vector representation.
- unsigned getNumElementsFlattened() const {
- return getNumRows() * getNumColumns();
- }
-
/// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
@@ -4658,12 +4647,12 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
}
void Profile(llvm::FoldingSetNodeID &ID) {
- Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
}
static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
- unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
unsigned Use, TypeClass TypeClass) {
ID.AddPointer(ElementType.getAsOpaquePtr());
ID.AddInteger(NumRows);
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index d134edc39e92ce..1d486ba3c20587 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -267,15 +267,15 @@ let Class = CooperativeMatrixType in {
def : Property<"elementType", QualType> {
let Read = [{ node->getElementType() }];
}
- def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
let Read = [{ node->getNumColumns() }];
}
- def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+ def : Property<"use", UInt32> { let Read = [{ node->getUse() }]; }
def : Creator<[{
- return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ return ctx.getCooperativeMatrixType(elementType, scope, numRows, numColumns, use);
}]>;
}
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 8a583ad504c706..56ce32cf3e7c5d 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 8ec1100f9f4d89..fd4f55e43f665d 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1155,7 +1155,7 @@ bool ASTStructuralEquivalence::isEquivalent(
const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
// The element types must be structurally equivalent and the number of rows
- // and columns must match.
+ // and columns and scope and use must match.
if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
Mat2->getElementType()) ||
Mat1->getScope() != Mat2->getScope() ||
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 7b3d239299c6e7..83bb35f1d886af 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,6 +632,30 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
+
#ifdef cl_intel_device_side_avc_motion_estimation
#define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0
@@ -771,27 +795,4 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
-typedef enum coop_matrix_operands_t {
- CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
- CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
-} coop_matrix_operands_t;
-
-typedef enum coop_matrix_use_t {
- CLK_COOPERATIVE_MATRIX_A = 0,
- CLK_COOPERATIVE_MATRIX_B = 1,
- CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
-} coop_matrix_use_t;
-
-typedef enum coop_matrix_layout_t {
- CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
- CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
-} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From 0bfa16ab62d7cf1d8eedfde484d4c649b51cd549 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sat, 12 Sep 2026 17:21:24 -0400
Subject: [PATCH 18/48] Add cooperative matrix type support in missing sites
---
clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp | 1 +
clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp | 1 +
clang/lib/CIR/CodeGen/CIRGenFunction.cpp | 2 ++
clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 2 ++
4 files changed, 6 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 7dd24a6d076e81..dc7c5671809722 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -536,6 +536,7 @@ mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 0a93ad586e2238..d320ad881cc463 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -648,6 +648,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLArrayRValue:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index 1fa57c4087da2c..2ef684e0aca331 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -102,6 +102,7 @@ cir::TypeEvaluationKind CIRGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -1733,6 +1734,7 @@ void CIRGenFunction::emitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index 9c952dd6054686..a7c2690856b3e6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -723,6 +723,7 @@ const char *vTableClassNameForType(const CIRGenModule &cgm, const Type *ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -1196,6 +1197,7 @@ mlir::Attribute CIRGenItaniumRTTIBuilder::buildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
>From de235d57cbc99ea4733bbc99b4c43cc65f513902 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 17 Sep 2026 21:44:28 -0400
Subject: [PATCH 19/48] Add cooperative matrix related changes to CIR
---
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 14 ++++++++++++++
clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 9 +++++++++
2 files changed, 23 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index a94139cff20ffb..1d06fdd688181d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -2290,6 +2290,20 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
case Builtin::BI__builtin_masked_store:
case Builtin::BI__builtin_masked_compress_store:
case Builtin::BI__builtin_masked_scatter:
+ case Builtin::BIcoop_mat_load:
+ case Builtin::BIcoop_mat_store:
+ case Builtin::BIcoop_mat_mulAdd:
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ case Builtin::BIcoop_mat_scalar_mul:
+ case Builtin::BIcoop_mat_scalar_neg:
+ case Builtin::BIcoop_mat_init:
+ case Builtin::BIcoop_mat_length:
+ case Builtin::BI__builtin_matrix_transpose:
+ case Builtin::BI__builtin_matrix_column_major_load:
+ case Builtin::BI__builtin_matrix_column_major_store:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 2082963025238c..5e44db0118f29d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -872,6 +872,10 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
}
mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
+ if (e->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ cgf.cgm.errorNYI(e->getSourceRange(), "cooperative matrix unary minus");
+ return {};
+ }
ignoreResultAssign = false;
mlir::Value operand;
if (!promotionType.isNull())
@@ -1236,6 +1240,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
mlir::Value VisitBin##OP(const BinaryOperator *e) { \
+ if (e->getType()->isCooperativeMatrixType()) { \
+ cgf.cgm.errorNYI(e->getSourceRange(), \
+ "cooperative matrix binary operator"); \
+ return {}; \
+ } \
QualType promotionTy = getPromotionType(e->getType()); \
auto result = emit##OP(emitBinOps(e, promotionTy)); \
if (result && !promotionTy.isNull()) \
>From 072a73b795a988edebd913b71c619a8abe781416 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 17 Sep 2026 23:45:47 -0400
Subject: [PATCH 20/48] Address review comments: 1. Update comments to remove
function names 2. Add change to emit error if extension is not enabled using
'pragma OPENCL_EXTENSION ... enable' in OpenCL source
---
.../include/clang/Basic/DiagnosticSemaKinds.td | 3 +++
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/include/clang/Sema/Sema.h | 9 ++++-----
clang/lib/AST/ASTContext.cpp | 3 +--
clang/lib/CodeGen/CodeGenFunction.h | 6 ++----
clang/lib/Sema/SemaType.cpp | 18 +++++++++++++++---
.../CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 ++
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 ++
.../coop-mat-init-length-codegen.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 ++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 7 +++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 2 ++
14 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 489faa0f818f92..4e4ccdc427474a 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12011,6 +12011,9 @@ def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
// OpenCL C Cooperative Matrix.
+def ext_coopmat_without_pragma
+ : Error<"cooperative matrix types require OpenCL extension '%0' to be enabled via pragma">;
+
def err_invalid_coopmat_attr
: Error<"invalid argument of cooperative matrix attribute">;
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 56ce32cf3e7c5d..8a583ad504c706 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 95ba15f9e0077c..41387270b41ebd 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8098,10 +8098,10 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
- // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
- // We allow casting between cooperative matrixes of the same scope, use, and
- // same dimensions i.e. when they have the same number of rows and columns.
- // Returns true if the cast is invalid.
+ /// Check type constraints for cooperative matrix casts. We allow casting
+ /// between cooperative matrixes of the same scope, use, and same dimensions
+ /// i.e. when they have the same number of rows and columns. Returns true if
+ /// the cast is invalid.
bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
@@ -15432,7 +15432,6 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
-
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
SourceLocation AttrLoc);
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 926e7269450a12..53feffe978a566 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -10830,8 +10830,7 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
-/// matrix types are compatible.
+/// Return true if the two specified cooperative matrix types are compatible.
static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 03380c36a7eb89..4b8586d169396a 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5540,13 +5540,11 @@ class CodeGenFunction : public CodeGenTypeCache {
llvm::ArrayRef<llvm::Value *> Args,
QualType ResultTy);
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
+ /// Emit the computation of the specified binary op, returning the result.
llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
llvm::Value *RHS, QualType ResultTy);
- /// EmitCoopMatNeg - Emit the computation of the negate op,
- /// returning the result.
+ /// Emit the computation of the negate op, returning the result.
llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 395e0ae920887e..da02adc20a68fd 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2681,8 +2681,19 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
return QualType();
}
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
+ auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ if (getLangOpts().OpenCL) {
+ if (!getOpenCLOptions().isSupported("cl_khr_cooperative_matrix",
+ getLangOpts()))
+ Diag(AttrLoc, diag::err_opencl_requires_extension)
+ << 0 << Result << "cl_khr_cooperative_matrix";
+ else if (!getOpenCLOptions().isAvailableOption("cl_khr_cooperative_matrix",
+ getLangOpts()))
+ Diag(AttrLoc, diag::ext_coopmat_without_pragma)
+ << "cl_khr_cooperative_matrix";
+ }
+ return Result;
}
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
@@ -9086,6 +9097,7 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
<< Attr << 2;
return;
}
+
Expr *RowsExpr = Attr.getArgAsExpr(0);
Expr *ColsExpr = Attr.getArgAsExpr(1);
QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
@@ -9093,7 +9105,7 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
CurType = T;
}
-/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+/// "coop_mat" attribute, like ext_vector_type
static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Sema &S) {
if (Attr.getNumArgs() != 4) {
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9d55bd8c61cc3c..c070010c24b499 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -19,6 +19,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Matrix type aliases used across tests
// ---------------------------------------------------------------------------
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index ad8df2939269f5..3dada9b4ef5de8 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -14,6 +14,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index dfa763b060e720..0b7a825b5d369c 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -5,6 +5,8 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Type definitions.
// ---------------------------------------------------------------------------
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index e8fad815806ece..38d98b28482bfe 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -33,6 +33,8 @@
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index 799f7a9e15a352..b32f6549deba50 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -15,6 +15,8 @@
// ---------------------------------------------------------------------------
// Type definitions shared across all tests.
// ---------------------------------------------------------------------------
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 0c27978d492a92..351728d2454943 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -12,6 +12,13 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+// ---------------------------------------------------------------------------
+// 0. Use of coop mat type without specifying pragma
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expected-error {{cooperative matrix types require OpenCL extension 'cl_khr_cooperative_matrix' to be enabled via pragma}}
+
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index bcedd94fdbf17d..dafa5b3bf123af 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -16,6 +16,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index f4c78b965d24aa..3f8b221c3edeb0 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -29,6 +29,8 @@
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
>From f486ee1f638d3a4e49fffc8bbb498e3e6c6a4698 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 18 Sep 2026 00:21:21 -0400
Subject: [PATCH 21/48] Address review comments:
1. Fix Matrix Scalar multiply for cases where LHS is scalar
2. Use getBuiltinID() to check for coop mat builtins instead of name
---
clang/lib/Sema/SemaDecl.cpp | 16 ++++++++++++++--
clang/lib/Sema/SemaExpr.cpp | 4 +++-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 4 ++++
3 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 24134fc65be2e5..e4ca43f475d978 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14147,9 +14147,21 @@ bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
if (!Fname)
return false;
- if (Fname->getName().starts_with("coop_mat") &&
- !Fname->getName().starts_with("coop_mat_length"))
+ switch (F->getBuiltinID()) {
+ case Builtin::BIcoop_mat_load:
+ case Builtin::BIcoop_mat_store:
+ case Builtin::BIcoop_mat_mulAdd:
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ case Builtin::BIcoop_mat_scalar_mul:
+ case Builtin::BIcoop_mat_scalar_neg:
+ case Builtin::BIcoop_mat_init:
return true;
+ default:
+ return false;
+ }
return false;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index af5cddd606ce1a..b66c764180967e 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16008,8 +16008,10 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType()) {
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (RHSTy->isCooperativeMatrixType() && LHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, RHSExpr, LHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 3dada9b4ef5de8..28f7660957a624 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -79,9 +79,13 @@ kernel void test_scalar_mul(__global float *ptr, float s) {
a = coop_mat_load(ptr, ROW_MAJOR, 16);
r = a * s;
(void)r;
+ MatA_t r2;
+ r2 = s * a;
+ (void)r2;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
>From 11f27b7ea403dca66213b4e2169cc3f9acda68b1 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 18 Sep 2026 02:24:09 -0400
Subject: [PATCH 22/48] Address more review comments: 1. Add documentation for
'coop_mat(...)' attribute 2. Add more checks for scope, use, and element type
3. Fix some typos
---
clang/include/clang/Basic/Attr.td | 2 +-
clang/include/clang/Basic/AttrDocs.td | 47 +++++++++++++++++++
.../clang/Basic/DiagnosticSemaKinds.td | 5 +-
clang/lib/Sema/SemaChecking.cpp | 15 +++---
clang/lib/Sema/SemaType.cpp | 22 ++++++++-
clang/test/AST/undocumented-attrs.cpp | 3 +-
6 files changed, 81 insertions(+), 13 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 38ed64bd369923..8c3df69009dcfe 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3986,7 +3986,7 @@ def CoopMatrixType : TypeAttr {
let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
- let Documentation = [Undocumented];
+ let Documentation = [CoopMatrixTypeDocs];
}
def Visibility : InheritableAttr {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index e8ce3c3dea7f3b..d47b29ffa1c03f 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1479,6 +1479,53 @@ See the OpenCL documentation for some more complete examples.
}];
}
+def CoopMatrixTypeDocs : Documentation {
+ let Category = DocCatType;
+ let Heading = "coop_mat";
+ let Content = [{
+The `coop_mat` attribute declares an OpenCL cooperative matrix type. Cooperative
+matrix operations are distributed across a set of invocations, such as a
+subgroup, allowing the implementation to optimize matrix multiply operations.
+
+The attribute is applied to a scalar numerical component type and takes four
+arguments:
+
+```c
+__attribute__((coop_mat(scope, rows, columns, use)))
+```
+- scope specifies the memory scope of the cooperative matrix.
+ It must be one of the supported OpenCL memory scopes, such as CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP.
+- rows specifies the number of rows in the matrix. It must be a constant expression.
+- columns specifies the number of columns in the matrix. It must be a constant expression.
+- use specifies the operand role of the matrix in coop_mat_muladd. It must be one of:
+ CLK_COOPERATIVE_MATRIX_A
+ CLK_COOPERATIVE_MATRIX_B
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+The coop_mat attribute can be used to declare local variables, function parameters, and function return values. Cooperative matrix types must not be used as kernel arguments.
+
+```c
+float vmat1 __attribute__((coop_mat(
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 8,
+ 8,
+ CLK_COOPERATIVE_MATRIX_A)));
+
+typedef half chmat44 __attribute__((coop_mat(
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 4,
+ 4,
+ CLK_COOPERATIVE_MATRIX_B)));
+chmat44 vmat2;
+```
+The base component type must be a scalar numerical type.
+Implicit and explicit casts between cooperative matrix types are not generally permitted.
+Conversion between cooperative matrix types is allowed only when the types have identical scope, dimensions, and use parameters, and their component types are compatible according to the OpenCL C conversion rules.
+Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_matmul_add, and coop_mat_length.
+See the OpenCL documentation for some more complete examples.
+ }];
+}
+
def DiagnoseIfDocs : Documentation {
let Category = DocCatFunction;
let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 4e4ccdc427474a..4b7cef01e95bee 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12049,9 +12049,12 @@ def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
"CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
-def err_coop_matrix_element_type
+def err_mismatched_coop_matrix_element_type
: Error<"inconsistent cooperative matrix element type">;
+def err_invalid_coop_matrix_element_type
+ : Error<"invalid cooperative matrix element type">;
+
def err_coop_matrix_row_or_col_mismatch
: Error<"mismatch of cooperative matrix row or column">;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 34baa40ebd3755..9a171cde215cd6 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17559,7 +17559,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
- if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
Arg0->getBeginLoc());
@@ -17594,7 +17594,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
if (MOutTy->getElementType().getUnqualifiedType() !=
M2Ty->getElementType().getUnqualifiedType())
- Diag(Loc, diag::err_coop_matrix_element_type);
+ Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
@@ -17623,7 +17623,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType()) {
- Diag(ALoc, diag::err_coop_matrix_element_type);
+ Diag(ALoc, diag::err_mismatched_coop_matrix_element_type);
return true;
}
return false;
@@ -17637,8 +17637,9 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
- CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ if (CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(),
+ Arg1->getType(), Arg1->getBeginLoc()))
+ return ExprError();
TheCall->setType(Arg0->getType());
@@ -17686,11 +17687,11 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType())
- return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
- return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+ return ExprError(Diag(Loc1, diag::err_mismatched_coop_matrix_element_type));
if (M0Ty->getNumRows() != M2Ty->getNumRows())
return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index da02adc20a68fd..057be06ce81524 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2668,17 +2668,35 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
}
std::optional<llvm::APSInt> ValueScope =
ScopeExpr->getIntegerConstantExpr(Context);
+ // Only the scope expression is invalid.
+ if (!ValueScope) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant
+ << ScopeExpr->getSourceRange();
+ return QualType();
+ }
unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
std::optional<llvm::APSInt> ValueUse =
UseExpr->getIntegerConstantExpr(Context);
+ // Only the use expression is invalid.
+ if (!ValueUse) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant
+ << UseExpr->getSourceRange();
+ return QualType();
+ }
unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Scope << "matrix scope";
return QualType();
}
if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Use << "matrix use";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
+ Diag(AttrLoc, diag::err_invalid_coop_matrix_element_type) << ElementTy;
return QualType();
}
auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index dff9d3845062a2..ed82ff3f517fdd 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,7 +27,6 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
-CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -89,4 +88,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 83
+CHECK-NEXT: Total: 82
\ No newline at end of file
>From 6ba891554260181477256cac9fc4e8b2659b3ae9 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 21 Sep 2026 12:24:29 -0400
Subject: [PATCH 23/48] Add missing checks for coop_mat builtins in
SemaChecking.cpp
1. Stride check in load/store
2. Check for optional matrix operand in muladd
---
.../clang/Basic/DiagnosticSemaKinds.td | 7 ++++++
clang/include/clang/Sema/Sema.h | 1 +
clang/lib/Sema/SemaChecking.cpp | 24 ++++++++++++++++++-
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 4b7cef01e95bee..0d9ed944642ef8 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12029,6 +12029,10 @@ def err_coop_mem_layout_enum
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
"CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+def err_coop_mat_stride_type
+ : Error<"stride argument for load/store of cooperative matrix must have "
+ "type 'size_t'">;
+
def err_coop_element_and_pointer_type
: Error<"inconsistent between cooperative matrix element type and buffer "
"pointer type">;
@@ -12036,6 +12040,9 @@ def err_coop_element_and_pointer_type
def err_coop_matrix_arg
: Error<"argument must be a valid cooperative matrix type">;
+def err_coop_matrix_operands_type
+ : Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
+
def err_coop_matrix_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 41387270b41ebd..b909bca97d0ece 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3120,6 +3120,7 @@ class Sema final : public SemaBase {
SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ bool CheckCoopMatrixLoadStoreStride(Expr *Stride);
ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9a171cde215cd6..9ba791ab4d942f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17542,6 +17542,14 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
+bool Sema::CheckCoopMatrixLoadStoreStride(Expr *Stride) {
+ if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
+ Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
+ return true;
+ }
+ return false;
+}
+
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
@@ -17550,6 +17558,8 @@ ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
return ExprError();
+ if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(2)))
+ return ExprError();
return CallResult;
}
@@ -17565,6 +17575,8 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
+ if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(3)))
+ return ExprError();
return CallResult;
}
@@ -17656,9 +17668,19 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
ExprResult CallResult) {
- if (checkArgCount(TheCall, 3))
+ if (checkArgCountRange(TheCall, 3, 4))
return ExprError();
+ // The fourth argument is optional. If present, it must be an enum.
+ if (TheCall->getNumArgs() == 4) {
+ Expr *Operands = TheCall->getArg(3);
+ QualType OperandsTy = Operands->getType().getCanonicalType();
+
+ if (!OperandsTy->isEnumeralType()) {
+ Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ }
+ }
+
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
Expr *Arg2 = TheCall->getArg(2);
>From f63b0defb5641a864371440e1bd0fbddfdbe2594 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 21 Sep 2026 13:45:20 -0400
Subject: [PATCH 24/48] Merge type lowering for cooperative matrix between
CGBuiltin.cpp and CodeGenTypes.cpp
---
clang/lib/CodeGen/CGBuiltin.cpp | 49 +++++++++++++-----------------
clang/lib/CodeGen/CodeGenTypes.cpp | 46 +++++++++++-----------------
clang/lib/CodeGen/CodeGenTypes.h | 5 +++
3 files changed, 43 insertions(+), 57 deletions(-)
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 5c39dcad6d58d2..d27958bacd16d8 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2744,20 +2744,13 @@ static std::string getSPIRVBuiltinName(unsigned BuiltinID,
return "";
}
-static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
- CodeGenModule &CGM,
+static llvm::TargetExtType *getTargetExtType(CodeGenModule &CGM,
const CooperativeMatrixType *MTy) {
- llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
- // Type arguments for TargetExtType
- llvm::Type *Tys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
- // Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *RetType = llvm::TargetExtType::get(
- CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
- return RetType;
+ llvm::Type *Ty = CGM.getTypes().ConvertTypeForCoopMat(MTy);
+ auto *TargetTy = llvm::dyn_cast<llvm::TargetExtType>(Ty);
+ if (!TargetTy)
+ return nullptr;
+ return TargetTy;
}
/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
@@ -4692,7 +4685,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Layout = EmitScalarExpr(E->getArg(1));
auto Stride = EmitScalarExpr(E->getArg(2));
// Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+ llvm::TargetExtType *RetType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
@@ -4728,7 +4721,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Layout = EmitScalarExpr(E->getArg(2));
auto Stride = EmitScalarExpr(E->getArg(3));
// Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ llvm::TargetExtType *ArgType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
@@ -4772,10 +4765,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
- auto *BType = getTargetExtType(*this, CGM, MBTy);
- auto *CType = getTargetExtType(*this, CGM, MCTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
+ auto *BType = getTargetExtType(CGM, MBTy);
+ auto *CType = getTargetExtType(CGM, MCTy);
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
@@ -4803,9 +4796,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
- auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
+ auto *BType = getTargetExtType(CGM, MBTy);
llvm::Type *ElTy = ConvertType(MTy->getElementType());
// Check if the data is signed/unsigned
QualType QT = MTy->getElementType();
@@ -4839,8 +4832,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg1 = EmitScalarExpr(E->getArg(1));
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
auto *BType = Arg1->getType();
// Check if the data is signed/unsigned
@@ -4870,8 +4863,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg0 = EmitScalarExpr(E->getArg(0));
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
// Check if the data is signed/unsigned
QualType QT = MTy->getElementType();
@@ -4899,7 +4892,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_init: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
auto Init = EmitScalarExpr(E->getArg(0));
- auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
@@ -4917,7 +4910,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_length: {
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 6ac24c6554922e..c5e4017b268f8d 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -90,6 +90,20 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
Ty->setName(OS.str());
}
+llvm::Type *CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
+ llvm::Type *ElementTy = ConvertType(CMT->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElementTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {CMT->getScope(), CMT->getNumRows(), CMT->getNumColumns(),
+ CMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
@@ -125,21 +139,8 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
- if (T->isCooperativeMatrixType()) {
- const CooperativeMatrixType *DMT =
- cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
- llvm::Type *ElTy = ConvertType(DMT->getElementType());
- // Type argument for TargetExtType
- llvm::Type *ArgTys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
- DMT->getUse()};
- // Create a TargetExtType to represent a Coop matrix type
- llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
- getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
- return CoopMatType;
- }
+ if (const CooperativeMatrixType *CMT = T->getAs<CooperativeMatrixType>())
+ return ConvertTypeForCoopMat(CMT);
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -719,20 +720,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
break;
}
case Type::CooperativeMatrix: {
- const CooperativeMatrixType *DMT =
- cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
- llvm::Type *ElTy = ConvertType(DMT->getElementType());
- // Type argument for TargetExtType
- llvm::Type *ArgTys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
- DMT->getUse()};
- // Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
- getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
-
- ResultType = CoopMatType;
+ ResultType = ConvertTypeForCoopMat(T->getAs<CooperativeMatrixType>());
break;
}
case Type::FunctionNoProto:
diff --git a/clang/lib/CodeGen/CodeGenTypes.h b/clang/lib/CodeGen/CodeGenTypes.h
index 860f1f1ccdce28..dfe0df716bbf39 100644
--- a/clang/lib/CodeGen/CodeGenTypes.h
+++ b/clang/lib/CodeGen/CodeGenTypes.h
@@ -126,6 +126,11 @@ class CodeGenTypes {
/// ConvertType - Convert type T into a llvm::Type.
llvm::Type *ConvertType(QualType T);
+ /// Convert a Clang cooperative matrix type to the corresponding LLVM type.
+ /// The element type and cooperative matrix properties, including scope,
+ /// dimensions, and use, are preserved in the resulting LLVM type.
+ llvm::Type *ConvertTypeForCoopMat(const CooperativeMatrixType *CMT);
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
>From 37c039009a5d4cfc14e3dcdc2953ee18434bbbf3 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 22 Sep 2026 12:22:50 -0400
Subject: [PATCH 25/48] Fix clang format issue
---
clang/lib/CodeGen/CodeGenTypes.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index c5e4017b268f8d..2147f06763ecba 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -90,7 +90,8 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
Ty->setName(OS.str());
}
-llvm::Type *CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
+llvm::Type *
+CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
llvm::Type *ElementTy = ConvertType(CMT->getElementType());
// Type arguments for TargetExtType
llvm::Type *Tys[] = {ElementTy};
>From b6c2945900f7b615c6f249b708e6247dc489ff18 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 23 Sep 2026 16:56:39 -0400
Subject: [PATCH 26/48] Add check for disallowing cooperative matrix type for
kernel parameters and update tests
---
clang/lib/Sema/SemaDecl.cpp | 3 +++
clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 10 ++++++++--
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 4 +++-
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 7 ++++++-
5 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index e4ca43f475d978..cd3a5f90352fb1 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -9901,6 +9901,9 @@ static bool isOpenCLSizeDependentType(ASTContext &C, QualType Ty) {
}
static OpenCLParamType getOpenCLKernelParameterType(Sema &S, QualType PT) {
+ if (PT->isCooperativeMatrixType())
+ return InvalidKernelParam;
+
if (PT->isDependentType())
return InvalidKernelParam;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index c070010c24b499..3ec349af595769 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -126,15 +126,21 @@ kernel void test_load_local(__local float *ptr) {
// Same matrix type, __global vs __local -> two distinct symbols.
// ===========================================================================
-kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+kernel void test_store_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_init(0.0f);
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
// CHECK-SAME: ptr addrspace(1)
-kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+kernel void test_store_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_init(0.0f);
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 28f7660957a624..112a4d58edab43 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -38,8 +38,10 @@ kernel void test_load(__global float *ptr) {
// ---------------------------------------------------------------------------
// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
-kernel void test_store(__global float *ptr, MatA_t a) {
+kernel void test_store(__global float *ptr) {
+ MatA_t a;
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index 38d98b28482bfe..a7737725e5dc70 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -41,7 +41,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
// that contains the vendor-extended "coop_mat" marker.
-kernel void test_mangling(MatA_t a) { (void)a; }
+void test_mangling(MatA_t a) { (void)a; }
// MANGLE: @{{.*}}test_mangling{{.*}}(
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 351728d2454943..9e7c992a63a255 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -19,7 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expecte
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t; // expected-note {{declared here}}
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
@@ -65,3 +65,8 @@ kernel void test_bad_assignment(__global float *ptr) {
bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
(void)bad;
}
+
+// ---------------------------------------------------------------------------
+// 5. Kernel parameter cannot be of cooperative matrix type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_kernel_param(MatA_t a) {} // expected-error {{cannot be used as the type of a kernel parameter}}
>From a9ccdf5b4ca01accc02abc7fe90cef31a6cf1806 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 23 Sep 2026 19:51:06 -0400
Subject: [PATCH 27/48] Update IsCoopMatBuiltin
---
clang/include/clang/Sema/Sema.h | 2 +-
clang/lib/Sema/SemaDecl.cpp | 7 +++----
clang/lib/Sema/SemaExpr.cpp | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
4 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index b909bca97d0ece..13bf3606a62140 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15461,7 +15461,7 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
- bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+ bool BuiltinReturnsCoopMatrix(Expr *RHSExpr);
/// Build a function type.
///
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index cd3a5f90352fb1..4b73b3f6b411f2 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14138,8 +14138,8 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if RHSExpr is a cooperative matrix builtin call.
-bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+// Return true if cooperative matrix builtin call returns cooperative matrix.
+bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
auto call = dyn_cast<CallExpr>(RHSExpr);
if (!call)
return false;
@@ -14152,7 +14152,6 @@ bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
return false;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
- case Builtin::BIcoop_mat_store:
case Builtin::BIcoop_mat_mulAdd:
case Builtin::BIcoop_mat_binary_add:
case Builtin::BIcoop_mat_binary_sub:
@@ -14690,7 +14689,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index b66c764180967e..419e765ad0b5e4 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16039,7 +16039,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
- if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(RHSExpr)) {
if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index dafa5b3bf123af..5de358100a5606 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -25,7 +25,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
// match the LHS variable type. Declare first, then assign so that
-// Sema's IsCoopMatrixBuiltin path fires correctly.
+// Sema's BuiltinReturnsCoopMatrix path fires correctly.
// ---------------------------------------------------------------------------
kernel void test_load_store(__global float *ptr,
__global float *out_ptr) {
>From c195c9f7d44399b8f8f2860ab67b3ba80938caa0 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 16:45:02 -0400
Subject: [PATCH 28/48] Remove dimension check for cooperative matrix
---
clang/include/clang/AST/TypeBase.h | 12 ------------
clang/lib/AST/ASTContext.cpp | 3 ---
2 files changed, 15 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 5328e2db64d2c4..b3437f3e7dae67 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4598,8 +4598,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
unsigned Scope;
unsigned Use;
- static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
-
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
@@ -4624,11 +4622,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns true if \p NumElements is a valid cooperative matrix dimension.
- static constexpr bool isDimensionValid(size_t NumElements) {
- return NumElements > 0 && NumElements <= MaxElementsPerDimension;
- }
-
/// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
@@ -4641,11 +4634,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum valid number of elements per dimension.
- static constexpr unsigned getMaxElementsPerDimension() {
- return MaxElementsPerDimension;
- }
-
void Profile(llvm::FoldingSetNodeID &ID) {
Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 53feffe978a566..0e646125658746 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4966,9 +4966,6 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
"need a valid element type");
- assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
- CooperativeMatrixType::isDimensionValid(NumColumns) &&
- "need valid matrix dimensions");
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
>From 4049f14c0e844a24c9d2a502526abf13752e8b2e Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 17:22:16 -0400
Subject: [PATCH 29/48] Update support for 'scope'
---
clang/include/clang/AST/TypeBase.h | 2 +-
clang/include/clang/Basic/AttrDocs.td | 6 ++--
clang/lib/Headers/opencl-c-base.h | 4 ---
.../coop-mat-codegen-mangling.cl | 32 +++++++++----------
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../coop-mat-init-length-codegen.cl | 6 ++--
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 +-
.../SemaOpenCL/coop-mat-init-length-sema.cl | 8 ++---
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 4 +--
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 2 +-
11 files changed, 33 insertions(+), 37 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index b3437f3e7dae67..957149d715fd39 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4624,7 +4624,7 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+ return Scope == 4; // memory_scope_sub_group
}
/// Returns true if \p Use is a valid cooperative matrix use.
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index d47b29ffa1c03f..67777076f2d01e 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1494,7 +1494,7 @@ arguments:
__attribute__((coop_mat(scope, rows, columns, use)))
```
- scope specifies the memory scope of the cooperative matrix.
- It must be one of the supported OpenCL memory scopes, such as CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP.
+ Only one of the OpenCL memory scopes (memory_scope_sub_group) is supported for cooperative matrix.
- rows specifies the number of rows in the matrix. It must be a constant expression.
- columns specifies the number of columns in the matrix. It must be a constant expression.
- use specifies the operand role of the matrix in coop_mat_muladd. It must be one of:
@@ -1506,13 +1506,13 @@ The coop_mat attribute can be used to declare local variables, function paramete
```c
float vmat1 __attribute__((coop_mat(
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ memory_scope_sub_group,
8,
8,
CLK_COOPERATIVE_MATRIX_A)));
typedef half chmat44 __attribute__((coop_mat(
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ memory_scope_sub_group,
4,
4,
CLK_COOPERATIVE_MATRIX_B)));
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 83bb35f1d886af..c4263b78652559 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,10 +632,6 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
typedef enum coop_matrix_operands_t {
CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 3ec349af595769..2446e41d4aec8d 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -13,7 +13,7 @@
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
@@ -165,18 +165,18 @@ kernel void test_no_cross_contamination(__global float *gptr,
// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
// Verify distinct declare lines exist (these appear at end of IR).
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc{{[0-9]+}}_16x16_u0_f32_sc{{[0-9]+}}_16x16_u1_f32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc{{[0-9]+}}_8x32_u0_f32_sc{{[0-9]+}}_32x8_u1_f32_sc{{[0-9]+}}_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc{{[0-9]+}}_16x16_u0_i32_sc{{[0-9]+}}_16x16_u1_i32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 112a4d58edab43..0bcb3d1d2ee378 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -8,7 +8,7 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 0b7a825b5d369c..ef61b8b40fbbf2 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -10,13 +10,13 @@
// ---------------------------------------------------------------------------
// Type definitions.
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
-typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index a7737725e5dc70..f16682934759b4 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -28,7 +28,7 @@
// expected-no-diagnostics
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index b32f6549deba50..a9ea17abb27308 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -17,16 +17,16 @@
// ---------------------------------------------------------------------------
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_B))) MatB_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
-typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 9e7c992a63a255..9315f3cfeb9537 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -6,7 +6,7 @@
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
@@ -24,7 +24,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
// ---------------------------------------------------------------------------
-// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// 1. Invalid scope value (0 is not memory_scope_sub_group)
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index 5de358100a5606..7bf1f5d2ae184c 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -10,7 +10,7 @@
// expected-no-diagnostics
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 3f8b221c3edeb0..7f7daab56ebcff 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -24,7 +24,7 @@
// ---------------------------------------------------------------------------
// Enum constants (from opencl-c-base.h via -finclude-default-header)
// ---------------------------------------------------------------------------
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
>From 8d9192d20593cceaeec0af2e801955569ffd4034 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 20:39:49 -0400
Subject: [PATCH 30/48] Update Semantic analysis to add missing checks
---
.../clang/Basic/DiagnosticSemaKinds.td | 11 +--
clang/include/clang/Sema/Sema.h | 10 +-
clang/lib/Sema/SemaChecking.cpp | 93 ++++++++++++++-----
clang/lib/Sema/SemaDecl.cpp | 14 +--
clang/lib/Sema/SemaExpr.cpp | 19 ++--
5 files changed, 90 insertions(+), 57 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 0d9ed944642ef8..04c83880a19b34 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12017,13 +12017,6 @@ def ext_coopmat_without_pragma
def err_invalid_coopmat_attr
: Error<"invalid argument of cooperative matrix attribute">;
-def err_coop_mat_not_primitive
- : Error<"cooperative matrix attribute only supported for integer and "
- "floating-point types">;
-
-def err_coop_mat_rol_col_size
- : Error<"cooperative matrix row and column must have power-of-two size">;
-
def err_coop_mem_layout_enum
: Error<"memory layout for cooperative matrix should be "
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
@@ -12074,8 +12067,8 @@ def err_unsupported_coopmat_binary_operator
def err_unsupported_coopmat_scalar_operator
: Error<"unsupported cooperative matrix scalar operator">;
-def err_ref_coopmat_var
- : Error<"invalid reference of cooperative matrix variable">;
+def err_invalid_operand_for_coopmat_scalar_operator
+ : Error<"unsupported type for cooperative matrix scalar operator">;
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 13bf3606a62140..ec8f2f389c9125 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3113,17 +3113,17 @@ class Sema final : public SemaBase {
bool BuiltinCountedByRef(CallExpr *TheCall);
// Coop matrix handling.
- void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ bool CheckCoopMatrixLoadElementType(QualType MatrixType,
SourceLocation MatrixLoc, CallExpr *call);
- void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ bool CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
QualType BufferType,
SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- bool CheckCoopMatrixLoadStoreStride(Expr *Stride);
+ bool CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx);
ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
- void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ bool CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
@@ -8055,7 +8055,7 @@ class Sema final : public SemaBase {
/// Are the two types cooperative matrix types and do they have the same
/// dimensions i.e. do they have the same number of rows and the same number
/// of columns? Also do they have the same scope and use?
- bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ bool areCoopMatrixTypesCompatible(QualType srcTy, QualType destTy);
bool areVectorTypesSameSize(QualType srcType, QualType destType);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9ba791ab4d942f..9721bf1f75daa2 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17491,35 +17491,38 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
}
// Check coop_mat_load/store matrix element has same type with buffer pointer.
-void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
QualType BufferType,
SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
- return;
+ return true;
}
assert(isa<PointerType>(BufferType));
auto *PTy = BufferType->castAs<PointerType>();
if (MTy->getElementType().getUnqualifiedType() !=
- PTy->getPointeeType().getUnqualifiedType())
+ PTy->getPointeeType().getUnqualifiedType()) {
Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+ return true;
+ }
+ return false;
}
-void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
SourceLocation MatrixLoc,
- CallExpr *call) {
-
- FunctionDecl *F = call->getDirectCallee();
+ CallExpr *Call) {
+ FunctionDecl *F = Call->getDirectCallee();
assert(F);
DeclarationName MemberName = F->getDeclName();
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (Fname->isStr("coop_mat_load"))
- CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ return CheckCoopMatrixLoadStoreElementType(MatrixType, Call->getArg(0)->getType(),
+ MatrixLoc);
+ return false;
}
// Check coop_mat_load/store layout argument
@@ -17542,11 +17545,14 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
-bool Sema::CheckCoopMatrixLoadStoreStride(Expr *Stride) {
+bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
+ assert(TheCall->getNumArgs() >= ArgIdx + 1);
+ Expr *Stride = TheCall->getArg(ArgIdx);
if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
return true;
}
+ TheCall->setArg(ArgIdx, Stride);
return false;
}
@@ -17558,7 +17564,7 @@ ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
return ExprError();
- if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(2)))
+ if (CheckCoopMatrixLoadStoreStride(TheCall, 2 /*ArgIdx*/))
return ExprError();
return CallResult;
}
@@ -17571,23 +17577,24 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
- CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc()))
+ return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
- if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(3)))
+ if (CheckCoopMatrixLoadStoreStride(TheCall, 3 /*ArgIdx*/))
return ExprError();
return CallResult;
}
-void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
FunctionDecl *F = TheCall->getDirectCallee();
assert(F);
DeclarationName MemberName = F->getDeclName();
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (!Fname->isStr("coop_mat_mulAdd"))
- return;
+ return false;
auto MC = TheCall->getArg(2);
auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
@@ -17599,17 +17606,24 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
if (!M2Ty)
Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
if (!MOutTy || !M2Ty)
- return;
+ return true;
- if (MOutTy->getUse() != 2)
+ if (MOutTy->getUse() != 2) {
Diag(Loc, diag::err_coop_matrix_useACC);
+ return true;
+ }
if (MOutTy->getElementType().getUnqualifiedType() !=
- M2Ty->getElementType().getUnqualifiedType())
+ M2Ty->getElementType().getUnqualifiedType()) {
Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
+ return true;
+ }
- if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesCompatible(TheCall->getType(), MC->getType())) {
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+ return false;
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
@@ -17623,7 +17637,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesCompatible(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17678,6 +17692,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!OperandsTy->isEnumeralType()) {
Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ return ExprError();
}
}
@@ -17700,12 +17715,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!M0Ty || !M1Ty || !M2Ty)
return ExprError();
- if (M0Ty->getUse() != 0)
+ if (M0Ty->getUse() != 0) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
- if (M1Ty->getUse() != 1)
+ return ExprError();
+ }
+ if (M1Ty->getUse() != 1) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
- if (M2Ty->getUse() != 2)
+ return ExprError();
+ }
+ if (M2Ty->getUse() != 2) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+ return ExprError();
+ }
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType())
@@ -17730,6 +17751,25 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+
+ if (!M0Ty) {
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ return ExprError();
+ }
+
+ Expr *Arg1 = TheCall->getArg(1);
+ QualType Ty = Arg1->getType();
+ if (!Ty->isScalarType()) {
+ Diag(Arg1->getBeginLoc(),
+ diag::err_invalid_operand_for_coopmat_scalar_operator);
+ return ExprError();
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ Ty.getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
TheCall->setType(Arg0->getType());
return CallResult;
@@ -17741,6 +17781,11 @@ ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ if (!M0Ty) {
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ return ExprError();
+ }
TheCall->setType(Arg0->getType());
return CallResult;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 4b73b3f6b411f2..2b41bbff0c6bff 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14138,18 +14138,14 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if cooperative matrix builtin call returns cooperative matrix.
+// Return true if builtin call returns cooperative matrix.
bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
- auto call = dyn_cast<CallExpr>(RHSExpr);
- if (!call)
+ auto Call = dyn_cast<CallExpr>(RHSExpr);
+ if (!Call)
return false;
- FunctionDecl *F = call->getDirectCallee();
+ FunctionDecl *F = Call->getDirectCallee();
if (!F)
return false;
- DeclarationName MemberName = F->getDeclName();
- IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
- if (!Fname)
- return false;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
case Builtin::BIcoop_mat_mulAdd:
@@ -14164,8 +14160,6 @@ bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
default:
return false;
}
-
- return false;
}
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 419e765ad0b5e4..93672667011a32 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -8100,8 +8100,7 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
-bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
- QualType destTy) {
+bool Sema::areCoopMatrixTypesCompatible(QualType srcTy, QualType destTy) {
if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
return false;
@@ -8232,7 +8231,7 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind) {
if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
- if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ if (!areCoopMatrixTypesCompatible(SrcTy, DestTy)) {
return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
<< DestTy << SrcTy << R;
}
@@ -16044,12 +16043,14 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
- auto call = dyn_cast<CallExpr>(RHSExpr);
- assert(call);
- call->setType(LHSExpr->getType());
- CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
- CheckCoopMatrixMatMulOutput(call);
+ auto Call = dyn_cast<CallExpr>(RHSExpr);
+ assert(Call);
+ Call->setType(LHSExpr->getType());
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ Call))
+ return ExprError();
+ if (CheckCoopMatrixMatMulOutput(Call))
+ return ExprError();
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
>From ea4636eaed3d18aa24302f5f8b3809fb38f3af53 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 20:42:05 -0400
Subject: [PATCH 31/48] Fix clang format issues
---
clang/lib/Sema/SemaChecking.cpp | 7 +++----
clang/lib/Sema/SemaExpr.cpp | 4 ++--
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9721bf1f75daa2..9bd7da3449774d 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17520,8 +17520,8 @@ bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (Fname->isStr("coop_mat_load"))
- return CheckCoopMatrixLoadStoreElementType(MatrixType, Call->getArg(0)->getType(),
- MatrixLoc);
+ return CheckCoopMatrixLoadStoreElementType(
+ MatrixType, Call->getArg(0)->getType(), MatrixLoc);
return false;
}
@@ -17767,8 +17767,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
}
- if (M0Ty->getElementType().getUnqualifiedType() !=
- Ty.getUnqualifiedType())
+ if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
TheCall->setType(Arg0->getType());
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 93672667011a32..83276b68374cfe 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16046,8 +16046,8 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
auto Call = dyn_cast<CallExpr>(RHSExpr);
assert(Call);
Call->setType(LHSExpr->getType());
- if (CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- Call))
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
+ LHSExpr->getBeginLoc(), Call))
return ExprError();
if (CheckCoopMatrixMatMulOutput(Call))
return ExprError();
>From ebc11e89c9001ed957bd350ed51858428842eaec Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 14:57:35 -0400
Subject: [PATCH 32/48] Fix issue caused by recent merge to the tip.
---
clang/lib/Sema/SemaChecking.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9bd7da3449774d..2e6b4e6bccae18 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17548,7 +17548,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
- if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
+ if (convertArgumentToType(Stride, Context.getSizeType())) {
Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
return true;
}
>From b1503cdb5cc35766beb5e5c542bfb54ff4380c19 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 16:35:51 -0400
Subject: [PATCH 33/48] Add missing check in TypePrinter.cpp
---
clang/lib/AST/TypePrinter.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 34380e3c8965f8..047ebdc71d7d98 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2178,6 +2178,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
case attr::ExtVectorType:
OS << "ext_vector_type";
break;
+ case attr::CoopMatrixType:
+ OS << "coop_mat";
+ break;
case attr::CFISalt:
OS << "cfi_salt(\"" << cast<CFISaltAttr>(T->getAttr())->getSalt() << "\")";
break;
>From 255c60e7e7c6efde0d388d7380152c012129f12f Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 18:07:29 -0400
Subject: [PATCH 34/48] Update test
---
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index f77fe542fd210f..10fd9c08173b58 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -31,10 +31,6 @@
// expected-no-diagnostics
void test_enum_constants(void) {
- // coop_matrix_scope_t
- coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
- (void)s;
-
// coop_matrix_use_t
coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
@@ -53,7 +49,7 @@ void test_enum_constants(void) {
// ── 2d. Enum constant values match the spec ──────────────────────────────────
void test_enum_values(void) {
- _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(memory_scope_sub_group == 4, "scope subgroup");
_Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
_Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
_Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
>From d55fbe244f243da35a163b05c2f78775a32bfbe9 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 29 Sep 2026 12:37:47 -0400
Subject: [PATCH 35/48] Fix minor build issue
---
clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index d89c80ca01754f..7a0b81494cf1e0 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -488,6 +488,7 @@ void ExprEngine::VisitCastExpr(const CastExpr *CastE, ExplodedNode *Pred,
// Various C++ casts that are not handled yet.
case CK_ToUnion:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_VectorSplat:
case CK_HLSLElementwiseCast:
case CK_HLSLAggregateSplatCast:
>From b62f40299199f4d54d224eed9400c871d8adcc76 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:06:37 -0400
Subject: [PATCH 36/48] Fix minor merge issue
---
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 1d06fdd688181d..b6bd8f1ca76417 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -2301,9 +2301,6 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
case Builtin::BIcoop_mat_scalar_neg:
case Builtin::BIcoop_mat_init:
case Builtin::BIcoop_mat_length:
- case Builtin::BI__builtin_matrix_transpose:
- case Builtin::BI__builtin_matrix_column_major_load:
- case Builtin::BI__builtin_matrix_column_major_store:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
>From ca7f193dd804813fe4751fa64c771b78c0032e82 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:07:46 -0400
Subject: [PATCH 37/48] Fix issue of direct initialization of cooperative
matrix
---
clang/lib/Sema/SemaDecl.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 2b41bbff0c6bff..f53442d9ceb25c 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14378,6 +14378,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
Init = Result.get();
}
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
+
// Perform the initialization.
bool InitializedFromParenListExpr = false;
bool IsParenListInit = false;
@@ -14680,19 +14693,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
-
- // Set return type of builtin call using type of LHS variable.
- // This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
- if (!VDecl->getType()->isCooperativeMatrixType()) {
- Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
- return;
- }
-
- auto call = dyn_cast<CallExpr>(Init);
- assert(call);
- call->setType(VDecl->getType());
- }
}
void Sema::ActOnInitializerError(Decl *D) {
>From c097f4eb4e4801f037ee3242d8d30f848476421c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:43:36 -0400
Subject: [PATCH 38/48] Update correct emission of MatrixOperand operand for
coop_mat_mulAdd
---
clang/include/clang/Basic/AttrDocs.td | 2 +-
clang/lib/CodeGen/CGBuiltin.cpp | 16 ++++++----------
2 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 67777076f2d01e..d3607d77105ac6 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1521,7 +1521,7 @@ chmat44 vmat2;
The base component type must be a scalar numerical type.
Implicit and explicit casts between cooperative matrix types are not generally permitted.
Conversion between cooperative matrix types is allowed only when the types have identical scope, dimensions, and use parameters, and their component types are compatible according to the OpenCL C conversion rules.
-Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_matmul_add, and coop_mat_length.
+Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_mat_mulAdd, coop_mat_length and element-wise arithmetic operations.
See the OpenCL documentation for some more complete examples.
}];
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d27958bacd16d8..4d5f8379b9e6f9 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -4750,27 +4750,23 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
auto Arg2 = E->getArg(2);
+ Value *Zero = Constant::getNullValue(Int32Ty);
auto MA = EmitScalarExpr(Arg0);
auto MB = EmitScalarExpr(Arg1);
auto MC = EmitScalarExpr(Arg2);
+ auto MatrixOperand =
+ (E->getNumArgs() == 4) ? EmitScalarExpr(E->getArg(3)) : Zero;
+
const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
- // Check if the data is signed/unsigned
- QualType QT = MATy->getElementType();
- bool isSigned = false;
- if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
- isSigned = BT->isSignedInteger();
- }
- llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
- llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
auto *RetType = getTargetExtType(CGM, MTy);
auto *AType = getTargetExtType(CGM, MATy);
auto *BType = getTargetExtType(CGM, MBTy);
auto *CType = getTargetExtType(CGM, MCTy);
llvm::FunctionType *FTy =
- llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int32Ty}, false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
@@ -4781,7 +4777,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
- auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, MatrixOperand});
NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
return RValue::get(NewCall);
}
>From f36505a3cc1fe5bc3ef9c30df42288fd657103ce Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 16:43:35 -0400
Subject: [PATCH 39/48] Remove unrelated verifier test and update some of the
sema checks.
---
clang/lib/Sema/SemaChecking.cpp | 15 ++++++--
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ---------------------
2 files changed, 12 insertions(+), 42 deletions(-)
delete mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 2e6b4e6bccae18..d9eedf672443a4 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17690,7 +17690,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
Expr *Operands = TheCall->getArg(3);
QualType OperandsTy = Operands->getType().getCanonicalType();
- if (!OperandsTy->isEnumeralType()) {
+ if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
return ExprError();
}
@@ -17728,9 +17728,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
return ExprError();
}
- if (M0Ty->getElementType().getUnqualifiedType() !=
- M1Ty->getElementType().getUnqualifiedType())
+ QualType M0ElemTy =
+ M0Ty->getElementType().getUnqualifiedType().getCanonicalType();
+ QualType M1ElemTy =
+ M1Ty->getElementType().getUnqualifiedType().getCanonicalType();
+ bool SameElementType = M0ElemTy == M1ElemTy;
+ bool SameIntegerWidth =
+ M0ElemTy->isIntegerType() && M1ElemTy->isIntegerType() &&
+ Context.getTypeSize(M0ElemTy) == Context.getTypeSize(M1ElemTy);
+
+ if (!SameElementType && !SameIntegerWidth) {
return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
+ }
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
deleted file mode 100644
index 7f2eedb657d41d..00000000000000
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ /dev/null
@@ -1,39 +0,0 @@
-; llvm/test/Verifier/coop_mat_verifier_fix.ll
-;
-; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
-;
-; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
-; ((A && B) <= 64). After the fix the parentheses are explicit.
-;
-; This test verifies that a valid @llvm.experimental.noalias.scope.decl
-; call with a 2-element scope list (no third argument) passes the verifier
-; without error, and that an invalid third argument (> 64 bits) is caught.
-;
-; ── 6a. Valid call — no third argument → verifier must accept ────────────────
-; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
-; RUN: llvm-as < %s | opt -passes=verify -disable-output
-;
-; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
-; RUN: llvm-as %s -o %t.bc
-; RUN: opt -passes=verify %t.bc -disable-output
-
-; VALID: @llvm.experimental.noalias.scope.decl
-
-declare void @llvm.experimental.noalias.scope.decl(metadata)
-
-define void @test_no_third_arg() {
- call void @llvm.experimental.noalias.scope.decl(
- metadata !0)
- ret void
-}
-
-; Third arg present and <= 64 bits wide — must also pass
-define void @test_valid_third_arg() {
- call void @llvm.experimental.noalias.scope.decl(
- metadata !0)
- ret void
-}
-
-!0 = !{!1}
-!1 = distinct !{!1, !2, !"scope_a"}
-!2 = distinct !{!2, !"domain_a"}
>From 53536526a07527911292a96eba33b517f25e5f2a Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 19:38:35 -0400
Subject: [PATCH 40/48] Update more sema checks
---
.../clang/Basic/DiagnosticSemaKinds.td | 8 +++---
clang/include/clang/Sema/Sema.h | 2 +-
clang/lib/Sema/SemaChecking.cpp | 9 ++----
clang/lib/Sema/SemaDecl.cpp | 28 +++++++++----------
clang/lib/Sema/SemaExpr.cpp | 24 ++++++++--------
5 files changed, 33 insertions(+), 38 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 04c83880a19b34..8531598debc9f3 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12036,7 +12036,7 @@ def err_coop_matrix_arg
def err_coop_matrix_operands_type
: Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
-def err_coop_matrix_assignment
+def err_invalid_coopmat_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
@@ -12058,9 +12058,6 @@ def err_invalid_coop_matrix_element_type
def err_coop_matrix_row_or_col_mismatch
: Error<"mismatch of cooperative matrix row or column">;
-def err_coop_matrix_use_type
- : Error<"inconsistent cooperative matrix use type">;
-
def err_unsupported_coopmat_binary_operator
: Error<"unsupported cooperative matrix binary operator">;
@@ -12070,6 +12067,9 @@ def err_unsupported_coopmat_scalar_operator
def err_invalid_operand_for_coopmat_scalar_operator
: Error<"unsupported type for cooperative matrix scalar operator">;
+def err_incompatible_coopmat_types
+ : Error<"incompatible cooperative matrix types">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index ec8f2f389c9125..5f3638a2ae03f0 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15461,7 +15461,7 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
- bool BuiltinReturnsCoopMatrix(Expr *RHSExpr);
+ CallExpr *getCoopMatrixBuiltinCall(Expr *RHSExpr);
/// Build a function type.
///
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index d9eedf672443a4..5cf5104d9146f4 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17620,7 +17620,7 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
if (!areCoopMatrixTypesCompatible(TheCall->getType(), MC->getType())) {
- Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+ Diag(Loc, diag::err_incompatible_coopmat_types);
return true;
}
return false;
@@ -17638,12 +17638,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
return true;
if (!areCoopMatrixTypesCompatible(ATy, BTy)) {
- Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
- return true;
- }
-
- if (M0Ty->getUse() != M1Ty->getUse()) {
- Diag(ALoc, diag::err_coop_matrix_use_type);
+ Diag(ALoc, diag::err_incompatible_coopmat_types);
return true;
}
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index f53442d9ceb25c..ae352bb3d51705 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14138,14 +14138,15 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if builtin call returns cooperative matrix.
-bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
+// Return the CallExpr if builtin call returns cooperative matrix.
+CallExpr *Sema::getCoopMatrixBuiltinCall(Expr *RHSExpr) {
+ RHSExpr = RHSExpr->IgnoreParens();
auto Call = dyn_cast<CallExpr>(RHSExpr);
if (!Call)
- return false;
+ return nullptr;
FunctionDecl *F = Call->getDirectCallee();
if (!F)
- return false;
+ return nullptr;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
case Builtin::BIcoop_mat_mulAdd:
@@ -14156,9 +14157,9 @@ bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
case Builtin::BIcoop_mat_scalar_mul:
case Builtin::BIcoop_mat_scalar_neg:
case Builtin::BIcoop_mat_init:
- return true;
+ return Call;
default:
- return false;
+ return nullptr;
}
}
@@ -14380,15 +14381,14 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
- if (!VDecl->getType()->isCooperativeMatrixType()) {
- Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
- return;
+ if (getLangOpts().OpenCL) {
+ if (CallExpr *Call = getCoopMatrixBuiltinCall(Init)) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_invalid_coopmat_assignment);
+ return;
+ }
+ Call->setType(VDecl->getType());
}
-
- auto call = dyn_cast<CallExpr>(Init);
- assert(call);
- call->setType(VDecl->getType());
}
// Perform the initialization.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 83276b68374cfe..75c507fc187914 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16038,19 +16038,19 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(RHSExpr)) {
- if (!LHSExpr->getType()->isCooperativeMatrixType()) {
- Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
- return ExprError();
+ if (getLangOpts().OpenCL) {
+ if (CallExpr *Call = getCoopMatrixBuiltinCall(RHSExpr)) {
+ if (!LHSExpr->getType()->isCooperativeMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_invalid_coopmat_assignment);
+ return ExprError();
+ }
+ Call->setType(LHSExpr->getType());
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
+ LHSExpr->getBeginLoc(), Call))
+ return ExprError();
+ if (CheckCoopMatrixMatMulOutput(Call))
+ return ExprError();
}
- auto Call = dyn_cast<CallExpr>(RHSExpr);
- assert(Call);
- Call->setType(LHSExpr->getType());
- if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
- LHSExpr->getBeginLoc(), Call))
- return ExprError();
- if (CheckCoopMatrixMatMulOutput(Call))
- return ExprError();
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
>From ac7f131fc99086fa24540c71347bf621363022ac Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 02:00:25 -0400
Subject: [PATCH 41/48] Update diagnostics and their use in Semantic analysis
and the associated tests
---
.../clang/Basic/DiagnosticSemaKinds.td | 40 ++--
clang/lib/Sema/SemaChecking.cpp | 73 +++----
clang/lib/Sema/SemaType.cpp | 8 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 178 ++++++++++++++++--
4 files changed, 219 insertions(+), 80 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8531598debc9f3..a80f764c1476d1 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12011,52 +12011,45 @@ def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
// OpenCL C Cooperative Matrix.
-def ext_coopmat_without_pragma
+def err_coopmat_without_pragma
: Error<"cooperative matrix types require OpenCL extension '%0' to be enabled via pragma">;
def err_invalid_coopmat_attr
- : Error<"invalid argument of cooperative matrix attribute">;
+ : Error<"invalid '%0' argument of cooperative matrix attribute">;
-def err_coop_mem_layout_enum
- : Error<"memory layout for cooperative matrix should be "
+def err_invalid_coopmat_mem_layout
+ : Error<"memory layout for cooperative matrix load/store should be "
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
"CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
-def err_coop_mat_stride_type
+def err_invalid_coopmat_stride_type
: Error<"stride argument for load/store of cooperative matrix must have "
"type 'size_t'">;
-def err_coop_element_and_pointer_type
+def err_coopmat_mismatch_element_and_pointer_type
: Error<"inconsistent between cooperative matrix element type and buffer "
"pointer type">;
-def err_coop_matrix_arg
+def err_invalid_coopmat_arg
: Error<"argument must be a valid cooperative matrix type">;
-def err_coop_matrix_operands_type
+def err_invalid_coopmat_memory_operand_type
: Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
def err_invalid_coopmat_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
-def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
- "CLK_COOPERATIVE_MATRIX_A">;
+def err_invalid_coopmat_use : Error<"argument of cooperative matrix must have '%0' use">;
-def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
- "CLK_COOPERATIVE_MATRIX_B">;
+def err_mismatched_coopmat_element_types
+ : Error<"inconsistent cooperative matrix element types">;
-def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
- "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+def err_invalid_coopmat_element_type
+ : Error<"%0 is invalid cooperative matrix element type">;
-def err_mismatched_coop_matrix_element_type
- : Error<"inconsistent cooperative matrix element type">;
-
-def err_invalid_coop_matrix_element_type
- : Error<"invalid cooperative matrix element type">;
-
-def err_coop_matrix_row_or_col_mismatch
- : Error<"mismatch of cooperative matrix row or column">;
+def err_coopmat_shapes_incompatible
+ : Error<"two cooperative matrices with incompatible shapes">;
def err_unsupported_coopmat_binary_operator
: Error<"unsupported cooperative matrix binary operator">;
@@ -12064,9 +12057,6 @@ def err_unsupported_coopmat_binary_operator
def err_unsupported_coopmat_scalar_operator
: Error<"unsupported cooperative matrix scalar operator">;
-def err_invalid_operand_for_coopmat_scalar_operator
- : Error<"unsupported type for cooperative matrix scalar operator">;
-
def err_incompatible_coopmat_types
: Error<"incompatible cooperative matrix types">;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 5cf5104d9146f4..6fe81a1ba3e623 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17496,7 +17496,7 @@ bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
- Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ Diag(MatrixLoc, diag::err_invalid_coopmat_arg);
return true;
}
@@ -17505,7 +17505,7 @@ bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
if (MTy->getElementType().getUnqualifiedType() !=
PTy->getPointeeType().getUnqualifiedType()) {
- Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+ Diag(MatrixLoc, diag::err_coopmat_mismatch_element_and_pointer_type);
return true;
}
return false;
@@ -17540,7 +17540,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
ArgError = true;
if (ArgError)
- Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+ Diag(LayoutExpr->getBeginLoc(), diag::err_invalid_coopmat_mem_layout);
return ArgError;
}
@@ -17548,8 +17548,16 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
- if (convertArgumentToType(Stride, Context.getSizeType())) {
- Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
+ QualType StrideType = Stride->getType();
+ QualType SizeType = Context.getSizeType();
+ bool IsSizeT = Context.hasSameType(StrideType, SizeType);
+ bool IsIntegerConstant = Stride->isIntegerConstantExpr(Context);
+ if (!IsSizeT && !IsIntegerConstant) {
+ Diag(Stride->getExprLoc(), diag::err_invalid_coopmat_stride_type);
+ return true;
+ }
+ if (convertArgumentToType(Stride, SizeType)) {
+ Diag(Stride->getExprLoc(), diag::err_invalid_coopmat_stride_type);
return true;
}
TheCall->setArg(ArgIdx, Stride);
@@ -17602,20 +17610,21 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
auto Loc = TheCall->getBeginLoc();
if (!MOutTy)
- Diag(Loc, diag::err_coop_matrix_arg);
+ Diag(Loc, diag::err_invalid_coopmat_arg);
if (!M2Ty)
- Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(MC->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!MOutTy || !M2Ty)
return true;
if (MOutTy->getUse() != 2) {
- Diag(Loc, diag::err_coop_matrix_useACC);
+ Diag(Loc, diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return true;
}
if (MOutTy->getElementType().getUnqualifiedType() !=
M2Ty->getElementType().getUnqualifiedType()) {
- Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
+ Diag(Loc, diag::err_mismatched_coopmat_element_types);
return true;
}
@@ -17631,9 +17640,9 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
- Diag(ALoc, diag::err_coop_matrix_arg);
+ Diag(ALoc, diag::err_invalid_coopmat_arg);
if (!M1Ty)
- Diag(BLoc, diag::err_coop_matrix_arg);
+ Diag(BLoc, diag::err_invalid_coopmat_arg);
if (!M0Ty || !M1Ty)
return true;
@@ -17644,7 +17653,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType()) {
- Diag(ALoc, diag::err_mismatched_coop_matrix_element_type);
+ Diag(ALoc, diag::err_mismatched_coopmat_element_types);
return true;
}
return false;
@@ -17686,7 +17695,8 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
QualType OperandsTy = Operands->getType().getCanonicalType();
if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
- Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ Diag(Operands->getBeginLoc(),
+ diag::err_invalid_coopmat_memory_operand_type);
return ExprError();
}
}
@@ -17702,24 +17712,27 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
auto Loc1 = Arg1->getBeginLoc();
if (!M0Ty)
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!M1Ty)
- Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg1->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!M2Ty)
- Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg2->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!M0Ty || !M1Ty || !M2Ty)
return ExprError();
if (M0Ty->getUse() != 0) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_A";
return ExprError();
}
if (M1Ty->getUse() != 1) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_B";
return ExprError();
}
if (M2Ty->getUse() != 2) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return ExprError();
}
@@ -17733,18 +17746,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
Context.getTypeSize(M0ElemTy) == Context.getTypeSize(M1ElemTy);
if (!SameElementType && !SameIntegerWidth) {
- return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coopmat_element_types));
}
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
- return ExprError(Diag(Loc1, diag::err_mismatched_coop_matrix_element_type));
+ return ExprError(Diag(Loc1, diag::err_mismatched_coopmat_element_types));
if (M0Ty->getNumRows() != M2Ty->getNumRows())
- return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ return ExprError(Diag(Loc0, diag::err_coopmat_shapes_incompatible));
if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
(M0Ty->getNumColumns() != M1Ty->getNumRows()))
- return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+ return ExprError(Diag(Loc1, diag::err_coopmat_shapes_incompatible));
return CallResult;
}
@@ -17759,20 +17772,12 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
auto Loc0 = Arg0->getBeginLoc();
if (!M0Ty) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
- return ExprError();
- }
-
- Expr *Arg1 = TheCall->getArg(1);
- QualType Ty = Arg1->getType();
- if (!Ty->isScalarType()) {
- Diag(Arg1->getBeginLoc(),
- diag::err_invalid_operand_for_coopmat_scalar_operator);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_arg);
return ExprError();
}
if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
- return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coopmat_element_types));
TheCall->setType(Arg0->getType());
return CallResult;
@@ -17786,7 +17791,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
Expr *Arg0 = TheCall->getArg(0);
auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
if (!M0Ty) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_arg);
return ExprError();
}
TheCall->setType(Arg0->getType());
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 057be06ce81524..59164f5afd0a1f 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2688,15 +2688,15 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Scope << "matrix scope";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << "scope";
return QualType();
}
if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Use << "matrix use";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << "use";
return QualType();
}
if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
- Diag(AttrLoc, diag::err_invalid_coop_matrix_element_type) << ElementTy;
+ Diag(AttrLoc, diag::err_invalid_coopmat_element_type) << ElementTy;
return QualType();
}
auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
@@ -2708,7 +2708,7 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
<< 0 << Result << "cl_khr_cooperative_matrix";
else if (!getOpenCLOptions().isAvailableOption("cl_khr_cooperative_matrix",
getLangOpts()))
- Diag(AttrLoc, diag::ext_coopmat_without_pragma)
+ Diag(AttrLoc, diag::err_coopmat_without_pragma)
<< "cl_khr_cooperative_matrix";
}
return Result;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 9315f3cfeb9537..d7b61d5d57b18f 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -1,6 +1,4 @@
-// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
-//
-// Patch 4: Sema — negative tests for diagnostic paths.
+// Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -13,7 +11,7 @@
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
// ---------------------------------------------------------------------------
-// 0. Use of coop mat type without specifying pragma
+// Use of cooperative matrix type without specifying pragma
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expected-error {{cooperative matrix types require OpenCL extension 'cl_khr_cooperative_matrix' to be enabled via pragma}}
@@ -24,21 +22,19 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
// ---------------------------------------------------------------------------
-// 1. Invalid scope value (0 is not memory_scope_sub_group)
+// Invalid scope value (0 is not memory_scope_sub_group)
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid 'scope' argument of cooperative matrix attribute}}
// ---------------------------------------------------------------------------
-// 2. Invalid use value (99 is not 0/1/2)
+// Invalid use value (99 is not 0/1/2)
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid 'use' argument of cooperative matrix attribute}}
// ---------------------------------------------------------------------------
-// 3. Mismatched element types in coop_mat_mulAdd
+// Mismatched element types in coop_mat_mulAdd
// a/b are float matrices, c is an int matrix — should fire element type
// mismatch diagnostic.
-// Use the two-step declare-then-assign pattern so that the coop_mat_load
-// fixup path fires correctly and we only get the intended mulAdd error.
// ---------------------------------------------------------------------------
kernel void test_muladd_type_mismatch(__global float *fptr,
__global int *iptr) {
@@ -50,15 +46,13 @@ kernel void test_muladd_type_mismatch(__global float *fptr,
c = coop_mat_load(iptr, ROW_MAJOR, 16);
MatC_int_t result;
- result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element types}}
(void)result;
}
// ---------------------------------------------------------------------------
-// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
-// "should be assigned to cooperative matrix type variable" diagnostic.
-// This intentionally uses single-step initialization because the error
-// fires precisely when the LHS is NOT a coop mat type.
+// Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
// ---------------------------------------------------------------------------
kernel void test_bad_assignment(__global float *ptr) {
float bad;
@@ -67,6 +61,156 @@ kernel void test_bad_assignment(__global float *ptr) {
}
// ---------------------------------------------------------------------------
-// 5. Kernel parameter cannot be of cooperative matrix type.
+// Kernel parameter cannot be of cooperative matrix type.
// ---------------------------------------------------------------------------
kernel void test_bad_kernel_param(MatA_t a) {} // expected-error {{cannot be used as the type of a kernel parameter}}
+
+// ---------------------------------------------------------------------------
+// Invalid memory layout for cooperative matrix load.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_load_layout(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, 99, 16); // expected-error {{memory layout for cooperative matrix load/store should be CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid memory layout for cooperative matrix store.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_store_layout(__global float *ptr) {
+ MatA_t a;
+ coop_mat_store(ptr, a, 99, 16); // expected-error {{memory layout for cooperative matrix load/store should be CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid stride type for cooperative matrix load.
+// The stride must have type size_t.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_load_stride(__global float *ptr) {
+ MatA_t a;
+ int stride = 16;
+ a = coop_mat_load(ptr, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid stride type for cooperative matrix store.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_store_stride(__global float *ptr) {
+ MatA_t a;
+ int stride = 16;
+ coop_mat_store(ptr, a, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
+}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrix element type must match the pointed-to buffer type.
+// ---------------------------------------------------------------------------
+kernel void test_load_element_pointer_mismatch(__global int *iptr) {
+ MatA_t a;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16); // expected-error {{inconsistent between cooperative matrix element type and buffer pointer type}}
+}
+
+// ---------------------------------------------------------------------------
+// The fourth coop_mat_mulAdd argument must have the cooperative-matrix
+// operands enum type.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_memory_operand_type() {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+
+ // The literal has type float rather than the required enum type.
+ (void)coop_mat_mulAdd(a, b, c, 16.2f); // expected-error {{fourth argument of 'coop_mat_mulAdd' must have enum type}}
+}
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd arguments must be cooperative matrix types.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_argument(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+
+ (void)coop_mat_mulAdd(a, b, 1.0f); // expected-error {{argument must be a valid cooperative matrix type}}
+}
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd requires the first operand to have use A.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_a_use(__global float *ptr) {
+ MatB_t b;
+ MatB_t b2;
+ MatC_int_t c;
+
+ (void)coop_mat_mulAdd(b, b2, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_A' use}}
+}
+
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd requires the second operand to have use B.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_b_use(__global float *ptr) {
+ MatA_t a;
+ MatA_t a2;
+ MatC_int_t c;
+
+ (void)coop_mat_mulAdd(a, a2, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_B' use}}
+}
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd requires the third operand to have accumulator use.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_accumulator_use(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatA_t c;
+
+ (void)coop_mat_mulAdd(a, b, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_ACCUMULATOR' use}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid cooperative matrix element type.
+// ---------------------------------------------------------------------------
+typedef bool __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatBadElement_t; // expected-error {{'bool' is invalid cooperative matrix element type}}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrices with incompatible shapes cannot be used together.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 8, 16, USE_B))) MatB8x16_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC16x16_t;
+
+kernel void test_incompatible_muladd_shapes() {
+ MatA_t a;
+ MatB8x16_t b;
+ MatC16x16_t c;
+
+ (void)coop_mat_mulAdd(a, b, c); // expected-error {{two cooperative matrices with incompatible shapes}}
+}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrix binary operators require supported operand types.
+// ---------------------------------------------------------------------------
+kernel void test_unsupported_coopmat_binary_operator() {
+ MatA_t a;
+ MatB_t b;
+
+ (void)(a & b); // expected-error {{unsupported cooperative matrix binary operator}}
+}
+
+// ---------------------------------------------------------------------------
+// Scalar cooperative-matrix operators are restricted to the supported
+// operator/type combinations.
+// ---------------------------------------------------------------------------
+kernel void test_unsupported_coopmat_scalar_operator() {
+ MatA_t a;
+
+ (void)(a / 2.0f); // expected-error {{unsupported cooperative matrix scalar operator}}
+}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrix assignment requires compatible cooperative matrix types.
+// ---------------------------------------------------------------------------
+kernel void test_incompatible_coopmat_assignment() {
+ MatA_t a;
+ MatB_t b;
+
+ a = b; // expected-error {{incompatible cooperative matrix types}}
+}
>From e9365e2e1b730fa80d6a9659bedbfae812c2dd00 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 12:05:11 -0400
Subject: [PATCH 42/48] Update tests
---
clang/test/AST/undocumented-attrs.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index ed82ff3f517fdd..525466b1efdc0c 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -88,4 +88,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 82
\ No newline at end of file
+CHECK-NEXT: Total: 82
>From fca41cc57cd7f07f03befec53b4fd2abd9c25914 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 13:00:28 -0400
Subject: [PATCH 43/48] Cleanup tests and one clang-format issue fix
---
clang/lib/Sema/SemaChecking.cpp | 20 ++++-----
.../coop-mat-codegen-mangling.cl | 9 +---
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 30 ++++++++-----
.../coop-mat-init-length-codegen.cl | 12 ++---
.../test/Preprocessor/coop-mat-opencl-ext.cl | 17 +++----
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 44 +++----------------
.../SemaOpenCL/coop-mat-init-length-sema.cl | 23 +++-------
clang/test/SemaOpenCL/coop-mat-sema.cl | 21 +++------
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 12 +----
9 files changed, 62 insertions(+), 126 deletions(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 6fe81a1ba3e623..538dacea790ea0 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17545,7 +17545,8 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
-bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
+bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall,
+ unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
QualType StrideType = Stride->getType();
@@ -17617,8 +17618,7 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
return true;
if (MOutTy->getUse() != 2) {
- Diag(Loc, diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
+ Diag(Loc, diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return true;
}
@@ -17695,8 +17695,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
QualType OperandsTy = Operands->getType().getCanonicalType();
if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
- Diag(Operands->getBeginLoc(),
- diag::err_invalid_coopmat_memory_operand_type);
+ Diag(Operands->getBeginLoc(), diag::err_invalid_coopmat_memory_operand_type);
return ExprError();
}
}
@@ -17721,18 +17720,15 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
return ExprError();
if (M0Ty->getUse() != 0) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_A";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_A";
return ExprError();
}
if (M1Ty->getUse() != 1) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_B";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_B";
return ExprError();
}
if (M2Ty->getUse() != 2) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return ExprError();
}
@@ -17776,6 +17772,8 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
}
+ Expr *Arg1 = TheCall->getArg(1);
+ QualType Ty = Arg1->getType();
if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
return ExprError(Diag(Loc0, diag::err_mismatched_coopmat_element_types));
TheCall->setType(Arg0->getType());
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 2446e41d4aec8d..3f08145c655e61 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -1,11 +1,4 @@
-// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
-//
-// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
-// Covers three mangling axes:
-// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
-// (B) coop_mat_mulAdd -- different element types (float vs int)
-// (C) coop_mat_load -- different pointer address spaces
-// (D) coop_mat_store -- different pointer address spaces
+// Tests for function-name mangling of SPIR-V friendly cooperative matrix intrinsics.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 0bcb3d1d2ee378..6eafdf44d5075c 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -1,7 +1,5 @@
-// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
-//
-// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
-// SPIR-V intrinsic names.
+// Cooperative matrix tests for SPIR-V friendly LLVM IR code generation from
+// clang frontend.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -21,7 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
-// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
// TargetExtType (visible in the call signature).
// ---------------------------------------------------------------------------
@@ -36,7 +34,19 @@ kernel void test_load(__global float *ptr) {
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
// ---------------------------------------------------------------------------
-// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// Same test as above. But 'a' is initialized directly using the 'load'.
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
kernel void test_store(__global float *ptr) {
MatA_t a;
@@ -47,7 +57,7 @@ kernel void test_store(__global float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
-// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
// ---------------------------------------------------------------------------
kernel void test_muladd(__global float *ptr) {
MatA_t a; MatB_t b; MatC_t c; MatC_t r;
@@ -61,7 +71,7 @@ kernel void test_muladd(__global float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
// ---------------------------------------------------------------------------
-// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// Binary add (float element) -> __spirv_CooperativeMatrixFAdd
// ---------------------------------------------------------------------------
kernel void test_binary_add(__global float *ptr) {
MatA_t a; MatA_t b; MatA_t r;
@@ -74,7 +84,7 @@ kernel void test_binary_add(__global float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
// ---------------------------------------------------------------------------
-// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
kernel void test_scalar_mul(__global float *ptr, float s) {
MatA_t a; MatA_t r;
@@ -90,7 +100,7 @@ kernel void test_scalar_mul(__global float *ptr, float s) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
-// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// Unary minus -> __spirv_CooperativeMatrixScalarNeg
// ---------------------------------------------------------------------------
kernel void test_unary_neg(__global float *ptr) {
MatA_t a; MatA_t r;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index ef61b8b40fbbf2..b7673d7cdf0f65 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -21,7 +21,7 @@ typedef int __attribute__((coop_mat(memory_scope_sub_group,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
// ---------------------------------------------------------------------------
-// 1. coop_mat_init — float scalar.
+// coop_mat_init — float scalar.
// Expected IR: call __spirv_CompositeConstruct with the scalar value,
// returning a spirv.CooperativeMatrixKHR TargetExtType.
// ---------------------------------------------------------------------------
@@ -37,7 +37,7 @@ kernel void test_init_float(global float *out) {
}
// ---------------------------------------------------------------------------
-// 2. coop_mat_init — integer scalar.
+// coop_mat_init — integer scalar.
// Expected IR: call __spirv_CompositeConstruct with an i32 value.
// ---------------------------------------------------------------------------
@@ -52,7 +52,7 @@ kernel void test_init_int(void) {
}
// ---------------------------------------------------------------------------
-// 3. coop_mat_init — zero initialisation.
+// coop_mat_init — zero initialisation.
// Expected IR: call __spirv_CompositeConstruct with 0.0.
// ---------------------------------------------------------------------------
@@ -67,7 +67,7 @@ kernel void test_init_zero(void) {
}
// ---------------------------------------------------------------------------
-// 4. coop_mat_length — returns unsigned int.
+// coop_mat_length — returns unsigned int.
// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
// an i32 alloca.
// ---------------------------------------------------------------------------
@@ -84,7 +84,7 @@ kernel void test_length(global unsigned int *out) {
}
// ---------------------------------------------------------------------------
-// 5. coop_mat_init then coop_mat_length — combined flow.
+// coop_mat_init then coop_mat_length — combined flow.
// Verifies the TargetExtType produced by init flows into length unchanged.
// ---------------------------------------------------------------------------
@@ -100,7 +100,7 @@ kernel void test_init_then_length(global unsigned int *out) {
}
// ---------------------------------------------------------------------------
-// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// coop_mat_length — calling convention is SPIR_FUNC.
// The call must NOT be a plain 'call' — it must be 'call spir_func'.
// ---------------------------------------------------------------------------
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 10fd9c08173b58..6d4d2bf43b4ea5 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,13 +1,6 @@
-// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+// cl_khr_cooperative_matrix registration test.
//
-// Patch 2: cl_khr_cooperative_matrix registration in
-// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
-//
-// Tests: extension macro is predefined when enabled, extension can be
-// explicitly enabled/disabled via pragma, all four enum types and
-// their constants are visible under -finclude-default-header.
-
-// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// Test if extension macro is predefined when the extension is enabled
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
@@ -15,7 +8,7 @@
// EXT: cl_khr_cooperative_matrix
-// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// Test if extension is NOT predefined when explicitly disabled
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
@@ -23,7 +16,7 @@
// NOEXT-NOT: cl_khr_cooperative_matrix
-// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// Enum constants are visible when extension is enabled
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -47,7 +40,7 @@ void test_enum_constants(void) {
(void)op;
}
-// ── 2d. Enum constant values match the spec ──────────────────────────────────
+// Enum constant values match the spec
void test_enum_values(void) {
_Static_assert(memory_scope_sub_group == 4, "scope subgroup");
_Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index f16682934759b4..4c0763a4d703c5 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -1,27 +1,10 @@
-// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
-//
-// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
-// ExprConstant, ItaniumMangle, MicrosoftMangle,
-// ASTReader/ASTWriter TypeLoc serialisation.
-//
-// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// Test Itanium mangling
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
-// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
-// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -O0 -include-pch %t.pch \
-// RUN: -ast-dump %t.pch_probe.cl \
-// RUN: | FileCheck %s --check-prefix=PCH
-//
-// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// Structural equivalence (no diagnostics on compatible pair)
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -39,30 +22,15 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
-// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
-// that contains the vendor-extended "coop_mat" marker.
+// Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
void test_mangling(MatA_t a) { (void)a; }
+// MANGLE: @{{.*}}test_mangling{{.*}}coop_mat(
-// MANGLE: @{{.*}}test_mangling{{.*}}(
-
-// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
-// must still carry a coop_mat parameter type.
-// PCH: FunctionDecl {{.*}} pch_probe
-// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
-
-// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+// Structural equivalence — produce no diagnostic.
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
-
void test_structural_equiv(void) {
MatA_t *p = 0;
MatA_alias *q = p; // same canonical type
(void)q;
}
-
-// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
-// integer, float, pointer …). Using __builtin_classify_type on it
-// compiles without error; result is 0 (no_type_class).
-void test_expr_constant(MatA_t a) {
- int cls = __builtin_classify_type(a);
- (void)cls;
-}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index a9ea17abb27308..5dee7d7a4bef51 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -1,16 +1,10 @@
// Tests for coop_mat_init and coop_mat_length — Sema (positive).
//
-// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
-// so arg-count and arg-type validation are not performed by Sema for these two
-// builtins. Only the positive (valid-code) path and the one assignment-target
-// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
-// tested here.
-//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
-// expected-no-diagnostics (sections 1-6 must compile cleanly)
+// expected-no-diagnostics
// ---------------------------------------------------------------------------
// Type definitions shared across all tests.
@@ -31,9 +25,7 @@ typedef int __attribute__((coop_mat(memory_scope_sub_group,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
// ---------------------------------------------------------------------------
-// 1. coop_mat_init: initialise each use role with a float scalar.
-// Two-step (declare then assign) is required because the return type is
-// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// coop_mat_init: initialise with a float scalar.
// ---------------------------------------------------------------------------
void test_coop_mat_init_basic(void) {
MatA_t a;
@@ -45,7 +37,7 @@ void test_coop_mat_init_basic(void) {
}
// ---------------------------------------------------------------------------
-// 2. coop_mat_init: integer element type.
+// coop_mat_init: integer element type.
// ---------------------------------------------------------------------------
void test_coop_mat_init_int(void) {
MatA_int_t m;
@@ -53,7 +45,7 @@ void test_coop_mat_init_int(void) {
}
// ---------------------------------------------------------------------------
-// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// coop_mat_init: re-initialise the same variable (chained assigns).
// ---------------------------------------------------------------------------
void test_coop_mat_init_chained(void) {
MatC_t acc;
@@ -62,8 +54,7 @@ void test_coop_mat_init_chained(void) {
}
// ---------------------------------------------------------------------------
-// 4. coop_mat_length: basic usage — returns unsigned int directly.
-// No two-step required for the length call itself.
+// coop_mat_length: basic usage — returns unsigned int directly.
// ---------------------------------------------------------------------------
void test_coop_mat_length_basic(void) {
MatA_t a;
@@ -73,7 +64,7 @@ void test_coop_mat_length_basic(void) {
}
// ---------------------------------------------------------------------------
-// 5. coop_mat_length: all three use roles.
+// coop_mat_length: all three use roles.
// ---------------------------------------------------------------------------
void test_coop_mat_length_roles(void) {
MatA_t a; a = coop_mat_init(0.0f);
@@ -86,7 +77,7 @@ void test_coop_mat_length_roles(void) {
}
// ---------------------------------------------------------------------------
-// 6. coop_mat_length: result used in arithmetic.
+// coop_mat_length: result used in arithmetic.
// ---------------------------------------------------------------------------
void test_coop_mat_length_arith(void) {
MatA_t a;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index 7bf1f5d2ae184c..63214bc07dc541 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -1,9 +1,5 @@
-// clang/test/SemaOpenCL/coop-mat-sema.cl
+// Tests for semantic checking etc.
//
-// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
-// builtin validation, expr handling, TreeTransform.
-//
-// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -23,9 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
-// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
-// match the LHS variable type. Declare first, then assign so that
-// Sema's BuiltinReturnsCoopMatrix path fires correctly.
+// load builtin test
// ---------------------------------------------------------------------------
kernel void test_load_store(__global float *ptr,
__global float *out_ptr) {
@@ -35,7 +29,7 @@ kernel void test_load_store(__global float *ptr,
}
// ---------------------------------------------------------------------------
-// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// mulAdd builtin test
// ---------------------------------------------------------------------------
kernel void test_muladd(__global float *ptr) {
MatA_t a;
@@ -50,7 +44,7 @@ kernel void test_muladd(__global float *ptr) {
}
// ---------------------------------------------------------------------------
-// 4d. Binary element-wise operators (+, -)
+// Binary element-wise operators (+, -) test
// ---------------------------------------------------------------------------
void test_binary_ops(MatA_t a, MatA_t b) {
MatA_t r_add = a + b;
@@ -59,7 +53,7 @@ void test_binary_ops(MatA_t a, MatA_t b) {
}
// ---------------------------------------------------------------------------
-// 4e. Scalar multiply operator
+// Scalar multiply operator test
// ---------------------------------------------------------------------------
void test_scalar_ops(MatA_t a, float s) {
MatA_t r = a * s;
@@ -67,7 +61,7 @@ void test_scalar_ops(MatA_t a, float s) {
}
// ---------------------------------------------------------------------------
-// 4f. Unary minus
+// Unary minus test
// ---------------------------------------------------------------------------
void test_unary_minus(MatA_t a) {
MatA_t r = -a;
@@ -75,8 +69,7 @@ void test_unary_minus(MatA_t a) {
}
// ---------------------------------------------------------------------------
-// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
-// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// Assignment test.
// ---------------------------------------------------------------------------
kernel void test_assignment(__global float *ptr) {
MatA_t a;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 7f7daab56ebcff..a519754346197b 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -1,11 +1,4 @@
-// clang/test/SemaOpenCL/coop_mat_type_infra.cl
-//
-// Patch 1: CooperativeMatrixType AST node and type infrastructure.
-//
-// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
-// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
-// RecursiveASTVisitor traversal, mergeTypes compatibility,
-// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+// CooperativeMatrixType AST node and type infrastructure tests
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -21,9 +14,6 @@
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
-// ---------------------------------------------------------------------------
-// Enum constants (from opencl-c-base.h via -finclude-default-header)
-// ---------------------------------------------------------------------------
#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
>From e40df841ca2db8bb8e0a6e24fc0224c53f8cdd0f Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 13:09:04 -0400
Subject: [PATCH 44/48] Fix clang-format issue
---
clang/lib/Sema/SemaChecking.cpp | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 538dacea790ea0..8ff9c1c4c72931 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17545,8 +17545,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
-bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall,
- unsigned ArgIdx) {
+bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
QualType StrideType = Stride->getType();
@@ -17618,7 +17617,8 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
return true;
if (MOutTy->getUse() != 2) {
- Diag(Loc, diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
+ Diag(Loc, diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return true;
}
@@ -17695,7 +17695,8 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
QualType OperandsTy = Operands->getType().getCanonicalType();
if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
- Diag(Operands->getBeginLoc(), diag::err_invalid_coopmat_memory_operand_type);
+ Diag(Operands->getBeginLoc(),
+ diag::err_invalid_coopmat_memory_operand_type);
return ExprError();
}
}
@@ -17720,15 +17721,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
return ExprError();
if (M0Ty->getUse() != 0) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_A";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_A";
return ExprError();
}
if (M1Ty->getUse() != 1) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_B";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_B";
return ExprError();
}
if (M2Ty->getUse() != 2) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return ExprError();
}
>From 40f14168e8e7200c38e45748a86e4143023d2498 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 14:06:54 -0400
Subject: [PATCH 45/48] Remove unnecessary test and fixed a minor bug in
function naming
---
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 8 ++++----
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 13 +------------
2 files changed, 5 insertions(+), 16 deletions(-)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 6eafdf44d5075c..27812082653304 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -23,12 +23,12 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
// TargetExtType (visible in the call signature).
// ---------------------------------------------------------------------------
-kernel void test_load(__global float *ptr) {
+kernel void test_load_1(__global float *ptr) {
MatA_t a;
a = coop_mat_load(ptr, ROW_MAJOR, 16);
(void)a;
}
-// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_1
// CHECK: call
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
@@ -36,11 +36,11 @@ kernel void test_load(__global float *ptr) {
// ---------------------------------------------------------------------------
// Same test as above. But 'a' is initialized directly using the 'load'.
// ---------------------------------------------------------------------------
-kernel void test_load(__global float *ptr) {
+kernel void test_load_2(__global float *ptr) {
MatA_t a = coop_mat_load(ptr, ROW_MAJOR, 16);
(void)a;
}
-// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_2
// CHECK: call
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index 4c0763a4d703c5..5649a52d03e590 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -1,10 +1,4 @@
-// Test Itanium mangling
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -emit-llvm -o - %s \
-// RUN: | FileCheck %s --check-prefix=MANGLE
-//
-// Structural equivalence (no diagnostics on compatible pair)
+// Test structural equivalence (no diagnostics on compatible pair)
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -22,11 +16,6 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
-// Mangling — function with a coop-mat parameter gets a mangled name
-// that contains the vendor-extended "coop_mat" marker.
-void test_mangling(MatA_t a) { (void)a; }
-// MANGLE: @{{.*}}test_mangling{{.*}}coop_mat(
-
// Structural equivalence — produce no diagnostic.
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
void test_structural_equiv(void) {
>From a29c03e65f7de7965f1a108aa461c3ad0f5f53b8 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 13 Oct 2026 17:09:21 -0400
Subject: [PATCH 46/48] Cleanup tests and add semantic checks for coop_mat_init
and coop_mat_length. Make sure 'coop_mat A = (coop_mat_load(...))' works.
---
.../clang/Basic/DiagnosticSemaKinds.td | 9 +-
clang/include/clang/Sema/Sema.h | 2 +
clang/lib/Sema/SemaChecking.cpp | 60 ++++++++++-
clang/lib/Sema/SemaDecl.cpp | 2 +
clang/lib/Sema/SemaExpr.cpp | 6 +-
.../coop-mat-codegen-mangling.cl | 40 +++-----
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 46 ++++-----
.../coop-mat-init-length-codegen.cl | 14 ---
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 25 -----
.../SemaOpenCL/coop-mat-init-length-sema.cl | 15 ---
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 99 +++++++------------
clang/test/SemaOpenCL/coop-mat-sema.cl | 78 ---------------
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 23 +----
13 files changed, 139 insertions(+), 280 deletions(-)
delete mode 100644 clang/test/SemaOpenCL/coop-mat-ast-utils.cl
delete mode 100644 clang/test/SemaOpenCL/coop-mat-sema.cl
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index a80f764c1476d1..38e25f99c96124 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12031,10 +12031,10 @@ def err_coopmat_mismatch_element_and_pointer_type
"pointer type">;
def err_invalid_coopmat_arg
- : Error<"argument must be a valid cooperative matrix type">;
+ : Error<"argument must be a valid cooperative matrix">;
-def err_invalid_coopmat_memory_operand_type
- : Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
+def err_invalid_coopmat_matrix_operand_type
+ : Error<"fourth argument of 'coop_mat_mulAdd' must have enum or integer type">;
def err_invalid_coopmat_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
@@ -12060,6 +12060,9 @@ def err_unsupported_coopmat_scalar_operator
def err_incompatible_coopmat_types
: Error<"incompatible cooperative matrix types">;
+def err_invalid_coopmat_builtin_arg
+ : Error<"invalid argument of cooperative matrix builtin">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5f3638a2ae03f0..c38b69d4a0ccec 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3137,6 +3137,8 @@ class Sema final : public SemaBase {
ExprResult CallResult);
ExprResult BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
ExprResult CallResult);
+ ExprResult BuiltinCoopMatInit(CallExpr *TheCall, ExprResult CallResult);
+ ExprResult BuiltinCoopMatLength(CallExpr *TheCall, ExprResult CallResult);
// Matrix builtin handling.
ExprResult BuiltinMatrixTranspose(CallExpr *TheCall, ExprResult CallResult);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 8ff9c1c4c72931..ccb322df7265c6 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4184,6 +4184,12 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
case Builtin::BIcoop_mat_scalar_neg:
return BuiltinCoopMatrixScalarUnaryOp(TheCall, TheCallResult);
+ case Builtin::BIcoop_mat_init:
+ return BuiltinCoopMatInit(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_length:
+ return BuiltinCoopMatLength(TheCall, TheCallResult);
+
case Builtin::BI__builtin_matrix_transpose:
return BuiltinMatrixTranspose(TheCall, TheCallResult);
@@ -17696,7 +17702,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
Diag(Operands->getBeginLoc(),
- diag::err_invalid_coopmat_memory_operand_type);
+ diag::err_invalid_coopmat_matrix_operand_type);
return ExprError();
}
}
@@ -17801,6 +17807,58 @@ ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
return CallResult;
}
+ExprResult Sema::BuiltinCoopMatInit(CallExpr *TheCall, ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg = TheCall->getArg(0);
+
+ // Defer validation for dependent arguments.
+ if (Arg->isTypeDependent())
+ return CallResult;
+
+ QualType ArgTy = Arg->getType();
+
+ // coop_mat_init(coop_mat)
+ if (ArgTy->isCooperativeMatrixType())
+ return CallResult;
+
+ // coop_mat_init(component_type)
+ if (ArgTy->isArithmeticType())
+ return CallResult;
+
+ // coop_mat_init(const gentype *)
+ if (const auto *PtrTy = ArgTy->getAs<PointerType>()) {
+ QualType PointeeTy = PtrTy->getPointeeType();
+
+ if (PointeeTy->isArithmeticType())
+ return CallResult;
+ }
+
+ Diag(Arg->getBeginLoc(), diag::err_invalid_coopmat_builtin_arg);
+ return ExprError();
+}
+
+ExprResult Sema::BuiltinCoopMatLength(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg = TheCall->getArg(0);
+
+ // Defer validation for dependent arguments.
+ if (Arg->isTypeDependent())
+ return CallResult;
+
+ if (!Arg->getType()->isCooperativeMatrixType()) {
+ Diag(Arg->getBeginLoc(), diag::err_invalid_coopmat_arg);
+ return ExprError();
+ }
+
+ // The builtin declaration should already provide the result type.
+ return CallResult;
+}
+
ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index ae352bb3d51705..a219080f244798 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14388,6 +14388,8 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
return;
}
Call->setType(VDecl->getType());
+ VDecl->setInit(Call);
+ return;
}
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 75c507fc187914..76fd0a33ebe067 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5193,8 +5193,7 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
}
// If the base is a matrix type, try to create a new MatrixSubscriptExpr.
- if (base->getType()->isMatrixType() ||
- base->getType()->isCooperativeMatrixType()) {
+ if (base->getType()->isMatrixType()) {
if (CheckAndReportCommaError(ArgExprs.front()))
return ExprError();
@@ -16039,12 +16038,13 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
if (getLangOpts().OpenCL) {
- if (CallExpr *Call = getCoopMatrixBuiltinCall(RHSExpr)) {
+ if (CallExpr *Call = getCoopMatrixBuiltinCall(RHSExpr->IgnoreParens())) {
if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_invalid_coopmat_assignment);
return ExprError();
}
Call->setType(LHSExpr->getType());
+ RHS.get()->setType(LHSExpr->getType());
if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
LHSExpr->getBeginLoc(), Call))
return ExprError();
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 3f08145c655e61..5ad339ae1e8c48 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -14,10 +14,6 @@
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-// ---------------------------------------------------------------------------
-// Matrix type aliases used across tests
-// ---------------------------------------------------------------------------
-
// --- float 16x16 (MulAdd group 1) ---
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatF_16x16_A;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatF_16x16_B;
@@ -33,11 +29,9 @@ typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatI_16x16_A;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatI_16x16_B;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatI_16x16_C;
-// ===========================================================================
-// (A) MulAdd: same element type (float), different matrix dimensions
-// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
-// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
-// ===========================================================================
+// MulAdd: same element type (float), different matrix dimensions
+// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
+// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
kernel void test_muladd_f32_16x16(__global float *ptr) {
MatF_16x16_A a;
@@ -71,10 +65,8 @@ kernel void test_muladd_f32_8x32(__global float *ptr) {
// CHECK-SAME: _f32_sc{{[0-9]+}}_32x8_u{{[0-9]+}}
// CHECK-SAME: _f32_sc{{[0-9]+}}_8x8_u{{[0-9]+}}
-// ===========================================================================
-// (B) MulAdd: different element types -- float 16x16 vs int 16x16
-// Same dimensions, different elem type -> must produce two distinct symbols.
-// ===========================================================================
+// MulAdd: different element types -- float 16x16 vs int 16x16
+// Same dimensions, different elem type -> must produce two distinct symbols.
kernel void test_muladd_i32_16x16(__global int *iptr) {
MatI_16x16_A a;
@@ -90,11 +82,9 @@ kernel void test_muladd_i32_16x16(__global int *iptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
// CHECK-SAME: _i32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
-// ===========================================================================
-// (C) Load: different pointer address spaces
-// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
-// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
-// ===========================================================================
+// Load: different pointer address spaces
+// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
+// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
kernel void test_load_global(__global float *ptr) {
MatF_16x16_A a;
@@ -114,11 +104,8 @@ kernel void test_load_local(__local float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
// CHECK-SAME: ptr addrspace(3)
-// ===========================================================================
-// (D) Store: different pointer address spaces
-// Same matrix type, __global vs __local -> two distinct symbols.
-// ===========================================================================
-
+// Store: different pointer address spaces
+// Same matrix type, __global vs __local -> two distinct symbols.
kernel void test_store_global(__global float *ptr) {
MatF_16x16_A a;
a = coop_mat_init(0.0f);
@@ -139,11 +126,8 @@ kernel void test_store_local(__local float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
// CHECK-SAME: ptr addrspace(3)
-// ===========================================================================
-// (E) Negative: confirm NO cross-contamination between address space variants
-// i.e. the _global load function is NOT called for the _local case.
-// ===========================================================================
-
+// Negative: confirm NO cross-contamination between address space variants
+// i.e. the _global load function is NOT called for the _local case.
kernel void test_no_cross_contamination(__global float *gptr,
__local float *lptr) {
MatF_16x16_A ag, al;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 27812082653304..73a935651ebb4a 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -6,6 +6,8 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
+// expected-no-diagnostics
+
#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
@@ -18,36 +20,36 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
-// ---------------------------------------------------------------------------
// coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
// TargetExtType (visible in the call signature).
-// ---------------------------------------------------------------------------
-kernel void test_load_1(__global float *ptr) {
- MatA_t a;
- a = coop_mat_load(ptr, ROW_MAJOR, 16);
- (void)a;
+kernel void test_load(__global float *ptr) {
+ MatA_t a_1;
+ a_1 = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a_1;
+ MatA_t a_2 = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a_2;
+ MatA_t a_3;
+ a_3 = (coop_mat_load(ptr, ROW_MAJOR, 16));
+ (void)a_3;
+ MatA_t a_4 = (coop_mat_load(ptr, ROW_MAJOR, 16));
+ (void)a_4;
}
-// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_1
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
// CHECK: call
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
-
-// ---------------------------------------------------------------------------
-// Same test as above. But 'a' is initialized directly using the 'load'.
-// ---------------------------------------------------------------------------
-kernel void test_load_2(__global float *ptr) {
- MatA_t a = coop_mat_load(ptr, ROW_MAJOR, 16);
- (void)a;
-}
-// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_2
// CHECK: call
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
-// ---------------------------------------------------------------------------
// coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
-// ---------------------------------------------------------------------------
kernel void test_store(__global float *ptr) {
MatA_t a;
coop_mat_store(ptr, a, ROW_MAJOR, 16);
@@ -56,9 +58,7 @@ kernel void test_store(__global float *ptr) {
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
-// ---------------------------------------------------------------------------
// coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
-// ---------------------------------------------------------------------------
kernel void test_muladd(__global float *ptr) {
MatA_t a; MatB_t b; MatC_t c; MatC_t r;
a = coop_mat_load(ptr, ROW_MAJOR, 16);
@@ -70,9 +70,7 @@ kernel void test_muladd(__global float *ptr) {
// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd
// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
-// ---------------------------------------------------------------------------
// Binary add (float element) -> __spirv_CooperativeMatrixFAdd
-// ---------------------------------------------------------------------------
kernel void test_binary_add(__global float *ptr) {
MatA_t a; MatA_t b; MatA_t r;
a = coop_mat_load(ptr, ROW_MAJOR, 16);
@@ -83,9 +81,7 @@ kernel void test_binary_add(__global float *ptr) {
// CHECK-LABEL: @__clang_ocl_kern_imp_test_binary_add
// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
-// ---------------------------------------------------------------------------
// Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
-// ---------------------------------------------------------------------------
kernel void test_scalar_mul(__global float *ptr, float s) {
MatA_t a; MatA_t r;
a = coop_mat_load(ptr, ROW_MAJOR, 16);
@@ -99,9 +95,7 @@ kernel void test_scalar_mul(__global float *ptr, float s) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
-// ---------------------------------------------------------------------------
// Unary minus -> __spirv_CooperativeMatrixScalarNeg
-// ---------------------------------------------------------------------------
kernel void test_unary_neg(__global float *ptr) {
MatA_t a; MatA_t r;
a = coop_mat_load(ptr, ROW_MAJOR, 16);
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index b7673d7cdf0f65..f5e73517eabfa7 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -7,9 +7,7 @@
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-// ---------------------------------------------------------------------------
// Type definitions.
-// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
@@ -20,11 +18,9 @@ typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
-// ---------------------------------------------------------------------------
// coop_mat_init — float scalar.
// Expected IR: call __spirv_CompositeConstruct with the scalar value,
// returning a spirv.CooperativeMatrixKHR TargetExtType.
-// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_init_float
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
@@ -36,10 +32,8 @@ kernel void test_init_float(global float *out) {
(void)a;
}
-// ---------------------------------------------------------------------------
// coop_mat_init — integer scalar.
// Expected IR: call __spirv_CompositeConstruct with an i32 value.
-// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_init_int
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
@@ -51,10 +45,8 @@ kernel void test_init_int(void) {
(void)m;
}
-// ---------------------------------------------------------------------------
// coop_mat_init — zero initialisation.
// Expected IR: call __spirv_CompositeConstruct with 0.0.
-// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_init_zero
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
@@ -66,11 +58,9 @@ kernel void test_init_zero(void) {
(void)acc;
}
-// ---------------------------------------------------------------------------
// coop_mat_length — returns unsigned int.
// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
// an i32 alloca.
-// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length
// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
@@ -83,10 +73,8 @@ kernel void test_length(global unsigned int *out) {
*out = len;
}
-// ---------------------------------------------------------------------------
// coop_mat_init then coop_mat_length — combined flow.
// Verifies the TargetExtType produced by init flows into length unchanged.
-// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_init_then_length
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
@@ -99,10 +87,8 @@ kernel void test_init_then_length(global unsigned int *out) {
*out = coop_mat_length(a);
}
-// ---------------------------------------------------------------------------
// coop_mat_length — calling convention is SPIR_FUNC.
// The call must NOT be a plain 'call' — it must be 'call spir_func'.
-// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length_cc
// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
deleted file mode 100644
index 5649a52d03e590..00000000000000
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ /dev/null
@@ -1,25 +0,0 @@
-// Test structural equivalence (no diagnostics on compatible pair)
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -fsyntax-only -verify %s
-
-// expected-no-diagnostics
-
-#define SCOPE memory_scope_sub_group
-#define USE_A CLK_COOPERATIVE_MATRIX_A
-#define USE_B CLK_COOPERATIVE_MATRIX_B
-#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
-
-#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
-
-// Structural equivalence — produce no diagnostic.
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
-void test_structural_equiv(void) {
- MatA_t *p = 0;
- MatA_alias *q = p; // same canonical type
- (void)q;
-}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index 5dee7d7a4bef51..98922aadd53e15 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -6,9 +6,6 @@
// expected-no-diagnostics
-// ---------------------------------------------------------------------------
-// Type definitions shared across all tests.
-// ---------------------------------------------------------------------------
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
typedef float __attribute__((coop_mat(memory_scope_sub_group,
@@ -24,9 +21,7 @@ typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
-// ---------------------------------------------------------------------------
// coop_mat_init: initialise with a float scalar.
-// ---------------------------------------------------------------------------
void test_coop_mat_init_basic(void) {
MatA_t a;
MatB_t b;
@@ -36,26 +31,20 @@ void test_coop_mat_init_basic(void) {
c = coop_mat_init(0.0f);
}
-// ---------------------------------------------------------------------------
// coop_mat_init: integer element type.
-// ---------------------------------------------------------------------------
void test_coop_mat_init_int(void) {
MatA_int_t m;
m = coop_mat_init(42);
}
-// ---------------------------------------------------------------------------
// coop_mat_init: re-initialise the same variable (chained assigns).
-// ---------------------------------------------------------------------------
void test_coop_mat_init_chained(void) {
MatC_t acc;
acc = coop_mat_init(0.0f);
acc = coop_mat_init(1.0f);
}
-// ---------------------------------------------------------------------------
// coop_mat_length: basic usage — returns unsigned int directly.
-// ---------------------------------------------------------------------------
void test_coop_mat_length_basic(void) {
MatA_t a;
a = coop_mat_init(0.0f);
@@ -63,9 +52,7 @@ void test_coop_mat_length_basic(void) {
(void)len;
}
-// ---------------------------------------------------------------------------
// coop_mat_length: all three use roles.
-// ---------------------------------------------------------------------------
void test_coop_mat_length_roles(void) {
MatA_t a; a = coop_mat_init(0.0f);
MatB_t b; b = coop_mat_init(0.0f);
@@ -76,9 +63,7 @@ void test_coop_mat_length_roles(void) {
(void)la; (void)lb; (void)lc;
}
-// ---------------------------------------------------------------------------
// coop_mat_length: result used in arithmetic.
-// ---------------------------------------------------------------------------
void test_coop_mat_length_arith(void) {
MatA_t a;
a = coop_mat_init(0.0f);
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index d7b61d5d57b18f..40b0aa2bfc483c 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -1,4 +1,4 @@
-// Sema — negative tests for diagnostic paths.
+// Negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -10,9 +10,7 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
-// ---------------------------------------------------------------------------
// Use of cooperative matrix type without specifying pragma
-// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expected-error {{cooperative matrix types require OpenCL extension 'cl_khr_cooperative_matrix' to be enabled via pragma}}
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
@@ -21,21 +19,13 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t; // expecte
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
-// ---------------------------------------------------------------------------
-// Invalid scope value (0 is not memory_scope_sub_group)
-// ---------------------------------------------------------------------------
+// Invalid 'scope' test
typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid 'scope' argument of cooperative matrix attribute}}
-// ---------------------------------------------------------------------------
-// Invalid use value (99 is not 0/1/2)
-// ---------------------------------------------------------------------------
+// Invalid 'use' test
typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid 'use' argument of cooperative matrix attribute}}
-// ---------------------------------------------------------------------------
// Mismatched element types in coop_mat_mulAdd
-// a/b are float matrices, c is an int matrix — should fire element type
-// mismatch diagnostic.
-// ---------------------------------------------------------------------------
kernel void test_muladd_type_mismatch(__global float *fptr,
__global int *iptr) {
MatA_t a;
@@ -48,93 +38,72 @@ kernel void test_muladd_type_mismatch(__global float *fptr,
MatC_int_t result;
result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element types}}
(void)result;
+
+ MatC_int_t result_2 = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element types}}
+ (void)result_2;
}
-// ---------------------------------------------------------------------------
-// Assignment of coop_mat_load result to a plain scalar — must fire the
-// "should be assigned to cooperative matrix type variable" diagnostic.
-// ---------------------------------------------------------------------------
+// Assignment of coop_mat_load result to a plain scalar
kernel void test_bad_assignment(__global float *ptr) {
float bad;
bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
(void)bad;
}
-// ---------------------------------------------------------------------------
// Kernel parameter cannot be of cooperative matrix type.
-// ---------------------------------------------------------------------------
kernel void test_bad_kernel_param(MatA_t a) {} // expected-error {{cannot be used as the type of a kernel parameter}}
-// ---------------------------------------------------------------------------
// Invalid memory layout for cooperative matrix load.
-// ---------------------------------------------------------------------------
kernel void test_invalid_load_layout(__global float *ptr) {
MatA_t a;
a = coop_mat_load(ptr, 99, 16); // expected-error {{memory layout for cooperative matrix load/store should be CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR}}
}
-// ---------------------------------------------------------------------------
// Invalid memory layout for cooperative matrix store.
-// ---------------------------------------------------------------------------
kernel void test_invalid_store_layout(__global float *ptr) {
MatA_t a;
coop_mat_store(ptr, a, 99, 16); // expected-error {{memory layout for cooperative matrix load/store should be CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR}}
}
-// ---------------------------------------------------------------------------
// Invalid stride type for cooperative matrix load.
-// The stride must have type size_t.
-// ---------------------------------------------------------------------------
kernel void test_invalid_load_stride(__global float *ptr) {
MatA_t a;
int stride = 16;
a = coop_mat_load(ptr, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
}
-// ---------------------------------------------------------------------------
// Invalid stride type for cooperative matrix store.
-// ---------------------------------------------------------------------------
kernel void test_invalid_store_stride(__global float *ptr) {
MatA_t a;
int stride = 16;
coop_mat_store(ptr, a, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
}
-// ---------------------------------------------------------------------------
// Cooperative matrix element type must match the pointed-to buffer type.
-// ---------------------------------------------------------------------------
kernel void test_load_element_pointer_mismatch(__global int *iptr) {
MatA_t a;
a = coop_mat_load(iptr, ROW_MAJOR, 16); // expected-error {{inconsistent between cooperative matrix element type and buffer pointer type}}
}
-// ---------------------------------------------------------------------------
-// The fourth coop_mat_mulAdd argument must have the cooperative-matrix
-// operands enum type.
-// ---------------------------------------------------------------------------
-kernel void test_invalid_muladd_memory_operand_type() {
+// Invalid value for fourth coop_mat_mulAdd argument.
+kernel void test_invalid_muladd_matrix_operand_type() {
MatA_t a;
MatB_t b;
MatC_int_t c;
- // The literal has type float rather than the required enum type.
- (void)coop_mat_mulAdd(a, b, c, 16.2f); // expected-error {{fourth argument of 'coop_mat_mulAdd' must have enum type}}
+ (void)coop_mat_mulAdd(a, b, c, 16.2f); // expected-error {{fourth argument of 'coop_mat_mulAdd' must have enum or integer type}}
}
-// ---------------------------------------------------------------------------
-// coop_mat_mulAdd arguments must be cooperative matrix types.
-// ---------------------------------------------------------------------------
+// Tests for first three arguments of coop_mat_mulAdd.
kernel void test_invalid_muladd_argument(__global float *ptr) {
MatA_t a;
MatB_t b;
MatC_int_t c;
- (void)coop_mat_mulAdd(a, b, 1.0f); // expected-error {{argument must be a valid cooperative matrix type}}
+ (void)coop_mat_mulAdd(a, b, 1.0f); // expected-error {{argument must be a valid cooperative matrix}}
}
-// ---------------------------------------------------------------------------
-// coop_mat_mulAdd requires the first operand to have use A.
-// ---------------------------------------------------------------------------
+// Tests for first argument of coop_mat_mulAdd.
kernel void test_invalid_muladd_a_use(__global float *ptr) {
MatB_t b;
MatB_t b2;
@@ -144,9 +113,7 @@ kernel void test_invalid_muladd_a_use(__global float *ptr) {
}
-// ---------------------------------------------------------------------------
-// coop_mat_mulAdd requires the second operand to have use B.
-// ---------------------------------------------------------------------------
+// Tests for second argument of coop_mat_mulAdd.
kernel void test_invalid_muladd_b_use(__global float *ptr) {
MatA_t a;
MatA_t a2;
@@ -155,9 +122,7 @@ kernel void test_invalid_muladd_b_use(__global float *ptr) {
(void)coop_mat_mulAdd(a, a2, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_B' use}}
}
-// ---------------------------------------------------------------------------
-// coop_mat_mulAdd requires the third operand to have accumulator use.
-// ---------------------------------------------------------------------------
+// Tests for third argument of coop_mat_mulAdd.
kernel void test_invalid_muladd_accumulator_use(__global float *ptr) {
MatA_t a;
MatB_t b;
@@ -166,14 +131,10 @@ kernel void test_invalid_muladd_accumulator_use(__global float *ptr) {
(void)coop_mat_mulAdd(a, b, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_ACCUMULATOR' use}}
}
-// ---------------------------------------------------------------------------
// Invalid cooperative matrix element type.
-// ---------------------------------------------------------------------------
typedef bool __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatBadElement_t; // expected-error {{'bool' is invalid cooperative matrix element type}}
-// ---------------------------------------------------------------------------
// Cooperative matrices with incompatible shapes cannot be used together.
-// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 8, 16, USE_B))) MatB8x16_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC16x16_t;
@@ -181,13 +142,10 @@ kernel void test_incompatible_muladd_shapes() {
MatA_t a;
MatB8x16_t b;
MatC16x16_t c;
-
(void)coop_mat_mulAdd(a, b, c); // expected-error {{two cooperative matrices with incompatible shapes}}
}
-// ---------------------------------------------------------------------------
-// Cooperative matrix binary operators require supported operand types.
-// ---------------------------------------------------------------------------
+// TEst of cooperative matrix binary operators.
kernel void test_unsupported_coopmat_binary_operator() {
MatA_t a;
MatB_t b;
@@ -195,22 +153,33 @@ kernel void test_unsupported_coopmat_binary_operator() {
(void)(a & b); // expected-error {{unsupported cooperative matrix binary operator}}
}
-// ---------------------------------------------------------------------------
-// Scalar cooperative-matrix operators are restricted to the supported
-// operator/type combinations.
-// ---------------------------------------------------------------------------
+// Test of scalar cooperative-matrix operators.
kernel void test_unsupported_coopmat_scalar_operator() {
MatA_t a;
(void)(a / 2.0f); // expected-error {{unsupported cooperative matrix scalar operator}}
}
-// ---------------------------------------------------------------------------
-// Cooperative matrix assignment requires compatible cooperative matrix types.
-// ---------------------------------------------------------------------------
+// Test of cooperative matrix assignment.
kernel void test_incompatible_coopmat_assignment() {
MatA_t a;
MatB_t b;
a = b; // expected-error {{incompatible cooperative matrix types}}
}
+
+// Test of coop_mat_init.
+void test_coop_mat_init_invalid(void) {
+ MatA_t a;
+ struct S {
+ int X;
+ };
+ struct S S;
+ a = coop_mat_init(&S); // expected-error {{invalid argument of cooperative matrix builtin}}
+}
+
+// Test of coop_mat_length.
+void test_coop_mat_length_invalid(void) {
+ int Value = 0;
+ int len = coop_mat_length(Value); // expected-error {{argument must be a valid cooperative matr}}
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
deleted file mode 100644
index 63214bc07dc541..00000000000000
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ /dev/null
@@ -1,78 +0,0 @@
-// Tests for semantic checking etc.
-//
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -fsyntax-only -verify %s
-
-// expected-no-diagnostics
-
-#define SCOPE memory_scope_sub_group
-#define USE_A CLK_COOPERATIVE_MATRIX_A
-#define USE_B CLK_COOPERATIVE_MATRIX_B
-#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
-#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
-
-#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
-
-// ---------------------------------------------------------------------------
-// load builtin test
-// ---------------------------------------------------------------------------
-kernel void test_load_store(__global float *ptr,
- __global float *out_ptr) {
- MatA_t a;
- a = coop_mat_load(ptr, ROW_MAJOR, 16);
- coop_mat_store(out_ptr, a, ROW_MAJOR, 16);
-}
-
-// ---------------------------------------------------------------------------
-// mulAdd builtin test
-// ---------------------------------------------------------------------------
-kernel void test_muladd(__global float *ptr) {
- MatA_t a;
- MatB_t b;
- MatC_t c;
- a = coop_mat_load(ptr, ROW_MAJOR, 16);
- b = coop_mat_load(ptr, ROW_MAJOR, 16);
- c = coop_mat_load(ptr, ROW_MAJOR, 16);
- MatC_t result;
- result = coop_mat_mulAdd(a, b, c);
- (void)result;
-}
-
-// ---------------------------------------------------------------------------
-// Binary element-wise operators (+, -) test
-// ---------------------------------------------------------------------------
-void test_binary_ops(MatA_t a, MatA_t b) {
- MatA_t r_add = a + b;
- MatA_t r_sub = a - b;
- (void)r_add; (void)r_sub;
-}
-
-// ---------------------------------------------------------------------------
-// Scalar multiply operator test
-// ---------------------------------------------------------------------------
-void test_scalar_ops(MatA_t a, float s) {
- MatA_t r = a * s;
- (void)r;
-}
-
-// ---------------------------------------------------------------------------
-// Unary minus test
-// ---------------------------------------------------------------------------
-void test_unary_minus(MatA_t a) {
- MatA_t r = -a;
- (void)r;
-}
-
-// ---------------------------------------------------------------------------
-// Assignment test.
-// ---------------------------------------------------------------------------
-kernel void test_assignment(__global float *ptr) {
- MatA_t a;
- a = coop_mat_load(ptr, ROW_MAJOR, 16);
- (void)a;
-}
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index a519754346197b..9af429fa115ff3 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -9,10 +9,6 @@
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
-// RUN: | FileCheck %s --check-prefix=PCH
#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
@@ -21,9 +17,7 @@
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-// ---------------------------------------------------------------------------
// Basic type construction — four use roles, float element type.
-// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_float16x16;
@@ -34,9 +28,7 @@ typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
// AST: TypedefDecl {{.*}} MatC_float16x16
// AST: TypedefDecl {{.*}} MatA_int8x8
-// ---------------------------------------------------------------------------
// Type printer — VarDecls carry the coop_mat attribute in their type string.
-// ---------------------------------------------------------------------------
void test_type_spelling(void) {
MatA_float16x16 a;
MatB_float16x16 b;
@@ -49,9 +41,7 @@ void test_type_spelling(void) {
// AST: VarDecl {{.*}} c {{.*}}coop_mat(
// AST: VarDecl {{.*}} d {{.*}}coop_mat(
-// ---------------------------------------------------------------------------
// Parameter / return type — type preserved through function boundary.
-// ---------------------------------------------------------------------------
MatA_float16x16 test_param_return(MatA_float16x16 in) {
return in;
}
@@ -59,9 +49,7 @@ MatA_float16x16 test_param_return(MatA_float16x16 in) {
// AST: FunctionDecl {{.*}} test_param_return
// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
-// ---------------------------------------------------------------------------
// TypeLoc operand traversal — all four operand slots populated.
-// ---------------------------------------------------------------------------
void test_typeloc_operands(void) {
float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
(void)local_acc;
@@ -69,28 +57,19 @@ void test_typeloc_operands(void) {
// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
-// ---------------------------------------------------------------------------
// RecursiveASTVisitor — element type (half) reachable through the node.
-// ---------------------------------------------------------------------------
void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
(void)x;
}
// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
-// ---------------------------------------------------------------------------
// mergeTypes / type compatibility — two identical typedefs resolve to the
// same canonical type; taking a pointer across them compiles cleanly.
-// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
void test_merge_types(void) {
MatA_float16x16 *p = 0;
MatA_alias *q = p; // same canonical type — no diagnostic
(void)q;
-}
-
-// ---------------------------------------------------------------------------
-// PCH serialisation round-trip.
-// ---------------------------------------------------------------------------
-// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
+}
\ No newline at end of file
>From de59679c9ef380f2bc5d2060ccd4de16065116ab Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 15 Oct 2026 15:58:01 -0400
Subject: [PATCH 47/48] Minor typo fix
---
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 40b0aa2bfc483c..9a2e102baec4bb 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -145,7 +145,7 @@ kernel void test_incompatible_muladd_shapes() {
(void)coop_mat_mulAdd(a, b, c); // expected-error {{two cooperative matrices with incompatible shapes}}
}
-// TEst of cooperative matrix binary operators.
+// Test of cooperative matrix binary operators.
kernel void test_unsupported_coopmat_binary_operator() {
MatA_t a;
MatB_t b;
>From 1b54ae32a02b5b3ac8df823f995b0a3e6a5677d1 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sat, 17 Oct 2026 13:35:19 -0400
Subject: [PATCH 48/48] Fix tests - Remove unnecessary checks for pointer type
in load/stores. Add check to report pointer to cooperative matrix.
---
clang/include/clang/AST/TypeBase.h | 3 ++
.../clang/Basic/DiagnosticSemaKinds.td | 3 ++
clang/include/clang/Sema/Sema.h | 5 ---
clang/lib/Sema/SemaChecking.cpp | 44 +------------------
clang/lib/Sema/SemaExpr.cpp | 3 --
clang/lib/Sema/SemaType.cpp | 5 +++
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 26 ++++++++---
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 11 +++--
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 10 -----
9 files changed, 37 insertions(+), 73 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 957149d715fd39..dd843fbcc32bf9 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4651,6 +4651,9 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
}
static bool isValidElementType(QualType ElemTy) {
+ // OpenCL vector types are valid when their scalar element type is valid.
+ if (const auto *VT = ElemTy->getAs<VectorType>())
+ return isValidElementType(VT->getElementType());
return ElemTy->isSpecificBuiltinType(BuiltinType::Char_S) ||
ElemTy->isSpecificBuiltinType(BuiltinType::UChar) ||
ElemTy->isSpecificBuiltinType(BuiltinType::Short) ||
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 38e25f99c96124..fc7225d2e8ae98 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12063,6 +12063,9 @@ def err_incompatible_coopmat_types
def err_invalid_coopmat_builtin_arg
: Error<"invalid argument of cooperative matrix builtin">;
+def err_coopmat_pointer
+ : Error<"pointer to cooperative matrix type is not allowed">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index c38b69d4a0ccec..0ac2e522815b4f 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3113,11 +3113,6 @@ class Sema final : public SemaBase {
bool BuiltinCountedByRef(CallExpr *TheCall);
// Coop matrix handling.
- bool CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc, CallExpr *call);
- bool CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
bool CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index ccb322df7265c6..86e7be167fed91 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17496,41 +17496,6 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
return ArgError;
}
-// Check coop_mat_load/store matrix element has same type with buffer pointer.
-bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc) {
- auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
- if (!MTy) {
- Diag(MatrixLoc, diag::err_invalid_coopmat_arg);
- return true;
- }
-
- assert(isa<PointerType>(BufferType));
- auto *PTy = BufferType->castAs<PointerType>();
-
- if (MTy->getElementType().getUnqualifiedType() !=
- PTy->getPointeeType().getUnqualifiedType()) {
- Diag(MatrixLoc, diag::err_coopmat_mismatch_element_and_pointer_type);
- return true;
- }
- return false;
-}
-
-bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc,
- CallExpr *Call) {
- FunctionDecl *F = Call->getDirectCallee();
- assert(F);
- DeclarationName MemberName = F->getDeclName();
- IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
- assert(Fname);
- if (Fname->isStr("coop_mat_load"))
- return CheckCoopMatrixLoadStoreElementType(
- MatrixType, Call->getArg(0)->getType(), MatrixLoc);
- return false;
-}
-
// Check coop_mat_load/store layout argument
bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
bool ArgError = false;
@@ -17587,13 +17552,8 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 4))
return ExprError();
- Expr *Arg0 = TheCall->getArg(0);
- Expr *Arg1 = TheCall->getArg(1);
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
- if (CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc()))
- return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
if (CheckCoopMatrixLoadStoreStride(TheCall, 3 /*ArgIdx*/))
@@ -17732,12 +17692,12 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
return ExprError();
}
if (M1Ty->getUse() != 1) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ Diag(Arg1->getBeginLoc(), diag::err_invalid_coopmat_use)
<< "CLK_COOPERATIVE_MATRIX_B";
return ExprError();
}
if (M2Ty->getUse() != 2) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ Diag(Arg2->getBeginLoc(), diag::err_invalid_coopmat_use)
<< "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return ExprError();
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 76fd0a33ebe067..c7414cade36953 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16045,9 +16045,6 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
}
Call->setType(LHSExpr->getType());
RHS.get()->setType(LHSExpr->getType());
- if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
- LHSExpr->getBeginLoc(), Call))
- return ExprError();
if (CheckCoopMatrixMatMulOutput(Call))
return ExprError();
}
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 59164f5afd0a1f..b6c55de47276b0 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1844,6 +1844,11 @@ static QualType deduceOpenCLPointeeAddrSpace(Sema &S, QualType PointeeType) {
QualType Sema::BuildPointerType(QualType T,
SourceLocation Loc, DeclarationName Entity) {
+ if (getLangOpts().OpenCL && T->getAs<CooperativeMatrixType>()) {
+ Diag(Loc, diag::err_coopmat_pointer);
+ return QualType();
+ }
+
if (T->isReferenceType()) {
// C++ 8.3.2p4: There shall be no ... pointers to references ...
Diag(Loc, diag::err_illegal_decl_pointer_to_reference)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 73a935651ebb4a..7b9589db70920c 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -6,8 +6,6 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
-// expected-no-diagnostics
-
#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
@@ -23,7 +21,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
// TargetExtType (visible in the call signature).
-kernel void test_load(__global float *ptr) {
+kernel void test_load(__global float *ptr, __global uchar4 *ptr4, __global char *ptrc) {
MatA_t a_1;
a_1 = coop_mat_load(ptr, ROW_MAJOR, 16);
(void)a_1;
@@ -34,6 +32,10 @@ kernel void test_load(__global float *ptr) {
(void)a_3;
MatA_t a_4 = (coop_mat_load(ptr, ROW_MAJOR, 16));
(void)a_4;
+ MatA_t a_5 = coop_mat_load(ptr4, ROW_MAJOR, 16);
+ (void)a_5;
+ MatA_t a_6 = coop_mat_load(ptrc, ROW_MAJOR, 64);
+ (void)a_6;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
// CHECK: call
@@ -48,15 +50,25 @@ kernel void test_load(__global float *ptr) {
// CHECK: call
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
// coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
-kernel void test_store(__global float *ptr) {
- MatA_t a;
- coop_mat_store(ptr, a, ROW_MAJOR, 16);
- (void)a;
+kernel void test_store(__global float *ptr, __global float4 *ptr4) {
+ MatA_t a_1;
+ coop_mat_store(ptr, a_1, ROW_MAJOR, 16);
+ (void)a_1;
+ MatA_t a_2;
+ coop_mat_store(ptr4, a_2, ROW_MAJOR, 16);
+ (void)a_2;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
// coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
kernel void test_muladd(__global float *ptr) {
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 9a2e102baec4bb..cb9fdea464d8d4 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -79,12 +79,6 @@ kernel void test_invalid_store_stride(__global float *ptr) {
coop_mat_store(ptr, a, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
}
-// Cooperative matrix element type must match the pointed-to buffer type.
-kernel void test_load_element_pointer_mismatch(__global int *iptr) {
- MatA_t a;
- a = coop_mat_load(iptr, ROW_MAJOR, 16); // expected-error {{inconsistent between cooperative matrix element type and buffer pointer type}}
-}
-
// Invalid value for fourth coop_mat_mulAdd argument.
kernel void test_invalid_muladd_matrix_operand_type() {
MatA_t a;
@@ -183,3 +177,8 @@ void test_coop_mat_length_invalid(void) {
int Value = 0;
int len = coop_mat_length(Value); // expected-error {{argument must be a valid cooperative matr}}
}
+
+// Test of pointer to coop_mat
+void test_pointer_to_coop_mat(void) {
+ MatA_t *p = 0; // expected-error {{pointer to cooperative matrix type is not allowed}}
+}
\ No newline at end of file
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 9af429fa115ff3..0bb651b3e52b33 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -63,13 +63,3 @@ void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))
}
// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
-
-// mergeTypes / type compatibility — two identical typedefs resolve to the
-// same canonical type; taking a pointer across them compiles cleanly.
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
-
-void test_merge_types(void) {
- MatA_float16x16 *p = 0;
- MatA_alias *q = p; // same canonical type — no diagnostic
- (void)q;
-}
\ No newline at end of file
More information about the cfe-commits
mailing list